From 70fd7c74b428bec8b469d05cc1c6116c5fa74085 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 27 Jun 2025 09:40:47 +0200 Subject: [PATCH 01/19] feat(errors): Improve error message for incorrect package UUID (#4270) (cherry picked from commit eefbef6491666b88400c94081258e2c41c9845c8) --- src/Operations.jl | 19 ++++++++++++++++++- test/new.jl | 6 +++--- test/pkg.jl | 12 ++++++++++++ 3 files changed, 33 insertions(+), 4 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index 0e8463abd8..31c33f6435 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -1570,7 +1570,24 @@ function check_registered(registries::Vector{Registry.RegistryInstance}, pkgs::V end pkg = is_all_registered(registries, pkgs) if pkg isa PackageSpec - pkgerror("expected package $(err_rep(pkg)) to be registered") + msg = "expected package $(err_rep(pkg)) to be registered" + # check if the name exists in the registry with a different uuid + if pkg.name !== nothing + reg_uuid = Pair{String, Vector{UUID}}[] + for reg in registries + uuids = Registry.uuids_from_name(reg, pkg.name) + if !isempty(uuids) + push!(reg_uuid, reg.name => uuids) + end + end + if !isempty(reg_uuid) + msg *= "\n You may have provided the wrong UUID for package $(pkg.name).\n Found the following UUIDs for that name:" + for (reg, uuids) in reg_uuid + msg *= "\n - $(join(uuids, ", ")) from registry: $reg" + end + end + end + pkgerror(msg) end return nothing end diff --git a/test/new.jl b/test/new.jl index 32899c621c..108fd7039f 100644 --- a/test/new.jl +++ b/test/new.jl @@ -452,7 +452,7 @@ end isolate(loaded_depot=true) do; mktempdir() do tempdir package_path = copy_test_package(tempdir, "UnregisteredUUID") Pkg.activate(package_path) - @test_throws PkgError("expected package `Example [142fd7e7]` to be registered") Pkg.add("JSON") + @test_throws PkgError Pkg.add("JSON") end end # empty git repo (no commits) isolate(loaded_depot=true) do; mktempdir() do tempdir @@ -1450,7 +1450,7 @@ end isolate(loaded_depot=true) do; mktempdir() do tempdir package_path = copy_test_package(tempdir, "UnregisteredUUID") Pkg.activate(package_path) - @test_throws PkgError("expected package `Example [142fd7e7]` to be registered") Pkg.update() + @test_throws PkgError Pkg.update() end end end @@ -1628,7 +1628,7 @@ end isolate(loaded_depot=true) do; mktempdir() do tempdir package_path = copy_test_package(tempdir, "UnregisteredUUID") Pkg.activate(package_path) - @test_throws PkgError("expected package `Example [142fd7e7]` to be registered") Pkg.update() + @test_throws PkgError Pkg.update() end end # package does not exist in the manifest isolate(loaded_depot=true) do diff --git a/test/pkg.jl b/test/pkg.jl index b33c9a22c1..bb09b8494d 100644 --- a/test/pkg.jl +++ b/test/pkg.jl @@ -207,6 +207,18 @@ temp_pkg_dir() do project_path @testset "package with wrong UUID" begin @test_throws PkgError Pkg.add(PackageSpec(TEST_PKG.name, UUID(UInt128(1)))) + @testset "package with wrong UUID but correct name" begin + try + Pkg.add(PackageSpec(name="Example", uuid=UUID(UInt128(2)))) + catch e + @test e isa PkgError + errstr = sprint(showerror, e) + @test occursin("expected package `Example [00000000]` to be registered", errstr) + @test occursin("You may have provided the wrong UUID for package Example.", errstr) + @test occursin("Found the following UUIDs for that name:", errstr) + @test occursin("- 7876af07-990d-54b4-ab0e-23690620f79a from registry: General", errstr) + end + end # Missing uuid @test_throws PkgError Pkg.add(PackageSpec(uuid = uuid4())) end From d3ad8d1a520ad0413b46fcbb37e62e5ad39f105f Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Fri, 27 Jun 2025 12:33:09 -0400 Subject: [PATCH 02/19] prompt for confirmation before removing compat entry (#4254) (cherry picked from commit 8b1f0b9ffe559aae285a951d81ae05ed543ea130) --- src/API.jl | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/src/API.jl b/src/API.jl index 21f7f846ee..78eaaa419d 100644 --- a/src/API.jl +++ b/src/API.jl @@ -1398,12 +1398,22 @@ function compat(ctx::Context, pkg::String, compat_str::Union{Nothing,String}; io io = something(io, ctx.io) pkg = pkg == "Julia" ? "julia" : pkg isnothing(compat_str) || (compat_str = string(strip(compat_str, '"'))) + existing_compat = Operations.get_compat_str(ctx.env.project, pkg) + # Double check before deleting a compat entry issue/3567 + if isinteractive() && (isnothing(compat_str) || isempty(compat_str)) + if !isnothing(existing_compat) + ans = Base.prompt(stdin, ctx.io, "No compat string was given. Delete existing compat entry `$pkg = $(repr(existing_compat))`? [y]/n", default = "y") + if lowercase(ans) !== "y" + return + end + end + end if haskey(ctx.env.project.deps, pkg) || pkg == "julia" success = Operations.set_compat(ctx.env.project, pkg, isnothing(compat_str) ? nothing : isempty(compat_str) ? nothing : compat_str) success === false && pkgerror("invalid compat version specifier \"$(compat_str)\"") write_env(ctx.env) if isnothing(compat_str) || isempty(compat_str) - printpkgstyle(io, :Compat, "entry removed for $(pkg)") + printpkgstyle(io, :Compat, "entry removed:\n $pkg = $(repr(existing_compat))") else printpkgstyle(io, :Compat, "entry set:\n $(pkg) = $(repr(compat_str))") end From 57baf3a0fa3e5413dfee7433f5ac93b1d5f262b2 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 11:19:18 +0200 Subject: [PATCH 03/19] fix what project file to look at when package without path but with a subdir is devved by name (#4271) (cherry picked from commit e9a05524087cbbbeb151e9b1c355008ef639ed06) --- src/Types.jl | 2 +- test/subdir.jl | 2 ++ 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/src/Types.jl b/src/Types.jl index a0a61586f2..a4c2b30524 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -804,7 +804,7 @@ function handle_repo_develop!(ctx::Context, pkg::PackageSpec, shared::Bool) new = true end if !has_uuid(pkg) - resolve_projectfile!(pkg, dev_path) + resolve_projectfile!(pkg, joinpath(dev_path, pkg.repo.subdir === nothing ? "" : pkg.repo.subdir)) end error_if_in_sysimage(pkg) pkg.path = shared ? dev_path : relative_project_path(ctx.env.manifest_file, dev_path) diff --git a/test/subdir.jl b/test/subdir.jl index cddf27992f..a9833d2829 100644 --- a/test/subdir.jl +++ b/test/subdir.jl @@ -241,6 +241,8 @@ end pkgstr("add $(packages_dir):dependencies/Dep") @test !isinstalled("Package") @test isinstalled("Dep") + pkg"dev Dep" # 4269 + @test isinstalled("Dep") pkg"rm Dep" # Add from path at branch. From 1ac99a416e17e1cfd067a5b670b057ecf63722c2 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 13:21:30 +0200 Subject: [PATCH 04/19] Fix leading whitespace in REPL commands with comma-separated packages (#4274) (cherry picked from commit d2e61025b775a785f4b6943ef408614ba460dcd0) --- src/REPLMode/REPLMode.jl | 4 ++-- test/repl.jl | 3 +++ 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/src/REPLMode/REPLMode.jl b/src/REPLMode/REPLMode.jl index 2abd52e5cc..b0c70745b6 100644 --- a/src/REPLMode/REPLMode.jl +++ b/src/REPLMode/REPLMode.jl @@ -223,7 +223,7 @@ function lex(cmd::String)::Vector{QString} return filter(x->!isempty(x.raw), qstrings) end -function tokenize(cmd::String) +function tokenize(cmd::AbstractString) cmd = replace(replace(cmd, "\r\n" => "; "), "\n" => "; ") # for multiline commands qstrings = lex(cmd) statements = foldl(qstrings; init=[QString[]]) do collection, next @@ -282,7 +282,7 @@ function core_parse(words::Vector{QString}; only_cmd=false) end parse(input::String) = - map(Base.Iterators.filter(!isempty, tokenize(input))) do words + map(Base.Iterators.filter(!isempty, tokenize(strip(input)))) do words statement, input_word = core_parse(words) statement.spec === nothing && pkgerror("`$input_word` is not a recognized command. Type ? for help with available commands") statement.options = map(parse_option, statement.options) diff --git a/test/repl.jl b/test/repl.jl index ce817eb20c..c1a82b56ad 100644 --- a/test/repl.jl +++ b/test/repl.jl @@ -66,6 +66,9 @@ temp_pkg_dir(;rm=false) do project_path; cd(project_path) do; pkg"rm Example Random" pkg"add Example,Random" pkg"rm Example,Random" + # Test leading whitespace handling (issue #4239) + pkg" add Example, Random" + pkg"rm Example Random" pkg"add Example#master" pkg"rm Example" pkg"add https://github.com/JuliaLang/Example.jl#master" From 5981c2e37a0b42f42f9929ca5ee2c2d69111a818 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 13:21:59 +0200 Subject: [PATCH 05/19] copy the app project instead of wrapping it (#4276) Co-authored-by: KristofferC (cherry picked from commit c78b40b3514a560a40bffada39743687fba38115) --- src/Apps/Apps.jl | 27 +++++++++++++++++++++++---- 1 file changed, 23 insertions(+), 4 deletions(-) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 61150bdaf5..33eeba8704 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -77,15 +77,33 @@ function _resolve(manifest::Manifest, pkgname=nothing) continue end + # TODO: Add support for existing manifest + projectfile = joinpath(app_env_folder(), pkg.name, "Project.toml") - # TODO: Add support for existing manifest + sourcepath = source_path(app_manifest_file(), pkg) + original_project_file = projectfile_path(sourcepath) + + mkpath(dirname(projectfile)) + + if isfile(original_project_file) + cp(original_project_file, projectfile; force=true) + chmod(projectfile, 0o644) # Make the copied project file writable + + # Add entryfile stanza pointing to the package entry file + # TODO: What if project file has its own entryfile? + project_data = TOML.parsefile(projectfile) + project_data["entryfile"] = joinpath(sourcepath, "src", "$(pkg.name).jl") + open(projectfile, "w") do io + TOML.print(io, project_data) + end + else + error("could not find project file for package $pkg") + end + # Create a manifest with the manifest entry Pkg.activate(joinpath(app_env_folder(), pkg.name)) do ctx = Context() - if isempty(ctx.env.project.deps) - ctx.env.project.deps[pkg.name] = uuid - end ctx.env.manifest.deps[uuid] = pkg Pkg.resolve(ctx) end @@ -153,6 +171,7 @@ function develop(pkg::PackageSpec) handle_package_input!(pkg) ctx = app_context() handle_repo_develop!(ctx, pkg, #=shared =# true) + Base.rm(joinpath(app_env_folder(), pkg.name); force=true, recursive=true) sourcepath = abspath(source_path(ctx.env.manifest_file, pkg)) project = get_project(sourcepath) From 1e8938f36a349bc3281c035a0ebb70364129c803 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 15:52:00 +0200 Subject: [PATCH 06/19] feat(apps): Add support for multiple apps per package via submodules (#4277) Co-authored-by: Claude (cherry picked from commit 25c2390edc0b1ea142361dcb2666e3dd56bedae0) --- docs/src/apps.md | 45 ++++++++++++++++++++++-- src/Apps/Apps.jl | 14 ++++---- src/Types.jl | 1 + src/manifest.jl | 8 ++++- src/project.jl | 3 +- test/apps.jl | 16 +++++++++ test/test_packages/Rot13.jl/Project.toml | 1 + test/test_packages/Rot13.jl/src/CLI.jl | 18 ++++++++++ test/test_packages/Rot13.jl/src/Rot13.jl | 2 ++ 9 files changed, 98 insertions(+), 10 deletions(-) create mode 100644 test/test_packages/Rot13.jl/src/CLI.jl diff --git a/docs/src/apps.md b/docs/src/apps.md index 1fccbb8fbd..083d68f0e1 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -7,7 +7,6 @@ - You need to manually make `~/.julia/bin` available on the PATH environment. - The path to the julia executable used is the same as the one used to install the app. If this julia installation gets removed, you might need to reinstall the app. - - You can only have one app installed per package. Apps are Julia packages that are intended to be run as a "standalone programs" (by e.g. typing the name of the app in the terminal possibly together with some arguments or flags/options). This is in contrast to most Julia packages that are used as "libraries" and are loaded by other files or in the Julia REPL. @@ -17,7 +16,7 @@ This is in contrast to most Julia packages that are used as "libraries" and are A Julia app is structured similar to a standard Julia library with the following additions: - A `@main` entry point in the package module (see the [Julia help on `@main`](https://docs.julialang.org/en/v1/manual/command-line-interface/#The-Main.main-entry-point) for details) -- An `[app]` section in the `Project.toml` file listing the executable names that the package provides. +- An `[apps]` section in the `Project.toml` file listing the executable names that the package provides. A very simple example of an app that prints the reversed input arguments would be: @@ -45,6 +44,48 @@ reverse = {} ``` The empty table `{}` is to allow for giving metadata about the app but it is currently unused. +## Multiple Apps per Package + +A single package can define multiple apps by using submodules. Each app can have its own entry point in a different submodule of the package. + +```julia +# src/MyMultiApp.jl +module MyMultiApp + +function (@main)(ARGS) + println("Main app: ", join(ARGS, " ")) +end + +include("CLI.jl") + +end # module +``` + +```julia +# src/CLI.jl +module CLI + +function (@main)(ARGS) + println("CLI submodule: ", join(ARGS, " ")) +end + +end # module CLI +``` + +```toml +# Project.toml + +# standard fields here + +[apps] +main-app = {} +cli-app = { submodule = "CLI" } +``` + +This will create two executables: +- `main-app` that runs `julia -m MyMultiApp` +- `cli-app` that runs `julia -m MyMultiApp.CLI` + After installing this app one could run: ``` diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 33eeba8704..f6afef54f9 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -353,9 +353,9 @@ function generate_shim(pkgname, app::AppInfo, env, julia) julia_bin_filename = joinpath(julia_bin_path(), filename) mkpath(dirname(filename)) content = if Sys.iswindows() - windows_shim(pkgname, julia, env) + windows_shim(pkgname, app, julia, env) else - bash_shim(pkgname, julia, env) + bash_shim(pkgname, app, julia, env) end overwrite_file_if_different(julia_bin_filename, content) if Sys.isunix() @@ -364,7 +364,8 @@ function generate_shim(pkgname, app::AppInfo, env, julia) end -function bash_shim(pkgname, julia::String, env) +function bash_shim(pkgname, app::AppInfo, julia::String, env) + module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" return """ #!/usr/bin/env bash @@ -374,12 +375,13 @@ function bash_shim(pkgname, julia::String, env) export JULIA_DEPOT_PATH=$(repr(join(DEPOT_PATH, ':'))) exec $julia \\ --startup-file=no \\ - -m $(pkgname) \\ + -m $(module_spec) \\ "\$@" """ end -function windows_shim(pkgname, julia::String, env) +function windows_shim(pkgname, app::AppInfo, julia::String, env) + module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" return """ @echo off @@ -391,7 +393,7 @@ function windows_shim(pkgname, julia::String, env) $julia ^ --startup-file=no ^ - -m $(pkgname) ^ + -m $(module_spec) ^ %* """ end diff --git a/src/Types.jl b/src/Types.jl index a4c2b30524..4706704e13 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -242,6 +242,7 @@ struct AppInfo name::String julia_command::Union{String, Nothing} julia_version::Union{VersionNumber, Nothing} + submodule::Union{String, Nothing} other::Dict{String,Any} end Base.@kwdef mutable struct Project diff --git a/src/manifest.jl b/src/manifest.jl index 945fe5a917..5eb1812106 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -86,9 +86,11 @@ read_apps(::Any) = pkgerror("Expected `apps` field to be a Dict") function read_apps(apps::Dict) appinfos = Dict{String, AppInfo}() for (appname, app) in apps + submodule = get(app, "submodule", nothing) appinfo = AppInfo(appname::String, app["julia_command"]::String, VersionNumber(app["julia_version"]::String), + submodule, app) appinfos[appinfo.name] = appinfo end @@ -334,7 +336,11 @@ function destructure(manifest::Manifest)::Dict for (appname, appinfo) in entry.apps julia_command = @something appinfo.julia_command joinpath(Sys.BINDIR, "julia" * (Sys.iswindows() ? ".exe" : "")) julia_version = @something appinfo.julia_version VERSION - new_entry["apps"][appname] = Dict{String,Any}("julia_command" => julia_command, "julia_version" => julia_version) + app_dict = Dict{String,Any}("julia_command" => julia_command, "julia_version" => julia_version) + if appinfo.submodule !== nothing + app_dict["submodule"] = appinfo.submodule + end + new_entry["apps"][appname] = app_dict end end if manifest.manifest_format.major == 1 diff --git a/src/project.jl b/src/project.jl index a856ddafe0..807bc61e98 100644 --- a/src/project.jl +++ b/src/project.jl @@ -82,7 +82,8 @@ function read_project_apps(raw::Dict{String,Any}, project::Project) info isa Dict{String,Any} || pkgerror(""" Expected value for app `$name` to be a dictionary. """) - appinfos[name] = AppInfo(name, nothing, nothing, other) + submodule = get(info, "submodule", nothing) + appinfos[name] = AppInfo(name, nothing, nothing, submodule, other) end return appinfos end diff --git a/test/apps.jl b/test/apps.jl index a7a21e4263..2704c5a4bb 100644 --- a/test/apps.jl +++ b/test/apps.jl @@ -12,11 +12,19 @@ isolate(loaded_depot=true) do Pkg.Apps.develop(path=joinpath(@__DIR__, "test_packages", "Rot13.jl")) current_path = ENV["PATH"] exename = Sys.iswindows() ? "juliarot13.bat" : "juliarot13" + cliexename = Sys.iswindows() ? "juliarot13cli.bat" : "juliarot13cli" withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do + # Test original app @test contains(Sys.which("$exename"), first(DEPOT_PATH)) @test read(`$exename test`, String) == "grfg\n" + + # Test submodule app + @test contains(Sys.which("$cliexename"), first(DEPOT_PATH)) + @test read(`$cliexename test`, String) == "CLI: grfg\n" + Pkg.Apps.rm("Rot13") @test Sys.which(exename) == nothing + @test Sys.which(cliexename) == nothing end end @@ -26,12 +34,20 @@ isolate(loaded_depot=true) do path = git_init_package(tmpdir, joinpath(@__DIR__, "test_packages", "Rot13.jl")) Pkg.Apps.add(path=path) exename = Sys.iswindows() ? "juliarot13.bat" : "juliarot13" + cliexename = Sys.iswindows() ? "juliarot13cli.bat" : "juliarot13cli" current_path = ENV["PATH"] withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do + # Test original app @test contains(Sys.which(exename), first(DEPOT_PATH)) @test read(`$exename test`, String) == "grfg\n" + + # Test submodule app + @test contains(Sys.which(cliexename), first(DEPOT_PATH)) + @test read(`$cliexename test`, String) == "CLI: grfg\n" + Pkg.Apps.rm("Rot13") @test Sys.which(exename) == nothing + @test Sys.which(cliexename) == nothing end # https://github.com/JuliaLang/Pkg.jl/issues/4258 diff --git a/test/test_packages/Rot13.jl/Project.toml b/test/test_packages/Rot13.jl/Project.toml index fe3ae6389c..a1933ed8a2 100644 --- a/test/test_packages/Rot13.jl/Project.toml +++ b/test/test_packages/Rot13.jl/Project.toml @@ -4,3 +4,4 @@ version = "0.1.0" [apps] juliarot13 = {} +juliarot13cli = { submodule = "CLI" } diff --git a/test/test_packages/Rot13.jl/src/CLI.jl b/test/test_packages/Rot13.jl/src/CLI.jl new file mode 100644 index 0000000000..342756b393 --- /dev/null +++ b/test/test_packages/Rot13.jl/src/CLI.jl @@ -0,0 +1,18 @@ +module CLI + +using ..Rot13: rot13 + +function (@main)(ARGS) + if length(ARGS) == 0 + println("Usage: rot13cli ") + return 1 + end + + for arg in ARGS + # Add a prefix to distinguish from main module output + println("CLI: $(rot13(arg))") + end + return 0 +end + +end # module CLI \ No newline at end of file diff --git a/test/test_packages/Rot13.jl/src/Rot13.jl b/test/test_packages/Rot13.jl/src/Rot13.jl index facbb92527..a97779dd77 100644 --- a/test/test_packages/Rot13.jl/src/Rot13.jl +++ b/test/test_packages/Rot13.jl/src/Rot13.jl @@ -14,4 +14,6 @@ function (@main)(ARGS) return 0 end +include("CLI.jl") + end # module Rot13 From 874690dae66c53a2f4f7429501e6f86280ce04db Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Wed, 2 Jul 2025 10:10:18 +0200 Subject: [PATCH 07/19] fix a header when cloning and use credentials when fetching (#4286) (cherry picked from commit ffdb668b81658f1c46041fc921fa03d2fdb9527a) --- src/GitTools.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/GitTools.jl b/src/GitTools.jl index 03cc08adff..562dada8ef 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -95,7 +95,7 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw @assert !isdir(source_path) || isempty(readdir(source_path)) url = normalize_url(url) printpkgstyle(io, :Cloning, header === nothing ? "git-repo `$url`" : header) - bar = MiniProgressBar(header = "Fetching:", color = Base.info_color()) + bar = MiniProgressBar(header = "Cloning:", color = Base.info_color()) fancyprint = can_fancyprint(io) fancyprint && start_progress(io, bar) if credentials === nothing @@ -122,7 +122,7 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw LibGit2.Callbacks() end mkpath(source_path) - return LibGit2.clone(url, source_path; callbacks=callbacks, credentials=credentials, kwargs...) + return LibGit2.clone(url, source_path; callbacks, credentials, kwargs...) end catch err rm(source_path; force=true, recursive=true) @@ -179,7 +179,7 @@ function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, end end else - return LibGit2.fetch(repo; remoteurl=remoteurl, callbacks=callbacks, refspecs=refspecs, kwargs...) + return LibGit2.fetch(repo; remoteurl, callbacks, credentials, refspecs, kwargs...) end catch err err isa LibGit2.GitError || rethrow() From 29555636671093edaa059d8bf5c38e0e554ef16c Mon Sep 17 00:00:00 2001 From: Glenn Moynihan Date: Wed, 2 Jul 2025 22:18:20 +0100 Subject: [PATCH 08/19] Fix bind_artifact! when platform has compare_strategy (#3624) Co-authored-by: Ian Butterworth (cherry picked from commit 799f7de320452f730bad5459ad97bee19ff25a65) --- src/Artifacts.jl | 2 +- test/artifacts.jl | 6 ++++++ 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/src/Artifacts.jl b/src/Artifacts.jl index 957d14aab9..12164b8df7 100644 --- a/src/Artifacts.jl +++ b/src/Artifacts.jl @@ -211,7 +211,7 @@ function bind_artifact!(artifacts_toml::String, name::String, hash::SHA1; meta = artifact_dict[name] if !isa(meta, Vector) error("Mapping for '$name' within $(artifacts_toml) already exists!") - elseif any(isequal(platform), unpack_platform(x, name, artifacts_toml) for x in meta) + elseif any(p -> platforms_match(platform, p), unpack_platform(x, name, artifacts_toml) for x in meta) error("Mapping for '$name'/$(triplet(platform)) within $(artifacts_toml) already exists!") end end diff --git a/test/artifacts.jl b/test/artifacts.jl index 605c3b26f8..5876ef5ab2 100644 --- a/test/artifacts.jl +++ b/test/artifacts.jl @@ -256,6 +256,12 @@ end @test ensure_artifact_installed("foo_txt", artifacts_toml; platform=linux64) == artifact_path(hash2) @test ensure_artifact_installed("foo_txt", artifacts_toml; platform=win32) == artifact_path(hash) + # Default HostPlatform() adds a compare_strategy key that doesn't get picked up from + # the Artifacts.toml + testhost = Platform("x86_64", "linux", Dict("libstdcxx_version" => "1.2.3")) + BinaryPlatforms.set_compare_strategy!(testhost, "libstdcxx_version", BinaryPlatforms.compare_version_cap) + @test_throws ErrorException bind_artifact!(artifacts_toml, "foo_txt", hash; download_info=download_info, platform=testhost) + # Next, check that we can get the download_info properly: meta = artifact_meta("foo_txt", artifacts_toml; platform=win32) @test meta["download"][1]["url"] == "http://google.com/hello_world" From c66d7ecf380af872c415429312d0181a44a047e3 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 3 Jul 2025 00:00:09 +0200 Subject: [PATCH 09/19] propagate `include_lazy` through `download_artifacts` (#3106) Co-authored-by: KristofferC Co-authored-by: Ian Butterworth (cherry picked from commit eab1b8b494197af291e20a73e4c5ed8e75b86555) --- src/Operations.jl | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/src/Operations.jl b/src/Operations.jl index 31c33f6435..eb14d797e7 100644 --- a/src/Operations.jl +++ b/src/Operations.jl @@ -818,7 +818,7 @@ function install_git( end end -function collect_artifacts(pkg_root::String; platform::AbstractPlatform=HostPlatform()) +function collect_artifacts(pkg_root::String; platform::AbstractPlatform=HostPlatform(), include_lazy::Bool=false) # Check to see if this package has an (Julia)Artifacts.toml artifacts_tomls = Tuple{String,Base.TOML.TOMLDict}[] for f in artifact_names @@ -842,7 +842,7 @@ function collect_artifacts(pkg_root::String; platform::AbstractPlatform=HostPlat end else # Otherwise, use the standard selector from `Artifacts` - artifacts = select_downloadable_artifacts(artifacts_toml; platform) + artifacts = select_downloadable_artifacts(artifacts_toml; platform, include_lazy) push!(artifacts_tomls, (artifacts_toml, artifacts)) end break @@ -862,7 +862,9 @@ end function download_artifacts(ctx::Context; platform::AbstractPlatform=HostPlatform(), julia_version = VERSION, - verbose::Bool=false) + verbose::Bool=false, + io::IO=stderr_f(), + include_lazy::Bool=false) env = ctx.env io = ctx.io fancyprint = can_fancyprint(io) @@ -888,7 +890,7 @@ function download_artifacts(ctx::Context; ansi_enablecursor = "\e[?25h" ansi_disablecursor = "\e[?25l" - all_collected_artifacts = reduce(vcat, map(pkg_root -> collect_artifacts(pkg_root; platform), pkg_roots)) + all_collected_artifacts = reduce(vcat, map(pkg_root -> collect_artifacts(pkg_root; platform, include_lazy), pkg_roots)) used_artifact_tomls = Set{String}(map(first, all_collected_artifacts)) longest_name_length = maximum(all_collected_artifacts; init=0) do (artifacts_toml, artifacts) maximum(textwidth, keys(artifacts); init=0) From 6ae29eda7a2e617ac2d88d90b540f24f6b8b8028 Mon Sep 17 00:00:00 2001 From: Jacob Quinn Date: Thu, 3 Jul 2025 08:18:22 -0600 Subject: [PATCH 10/19] Change refs/* to refs/heads/* to speed up repo cloning w/ many branches (#2330) Co-authored-by: Ian Butterworth Co-authored-by: KristofferC (cherry picked from commit 7381ccf45a93674ce94f054a465e7132f50f5092) --- src/Types.jl | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/Types.jl b/src/Types.jl index 4706704e13..1ead221dc2 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -699,7 +699,7 @@ function read_package(path::String) return project end -const refspecs = ["+refs/*:refs/remotes/cache/*"] +const refspecs = ["+refs/heads/*:refs/remotes/cache/heads/*"] function relative_project_path(project_file::String, path::String) # compute path relative the project From 6f8966741d49502f94fabb50b9c4a053fcbe05c7 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 3 Jul 2025 16:19:54 +0200 Subject: [PATCH 11/19] also use a bare repo when cloning for `add` using cli GIT (#4296) Co-authored-by: KristofferC (cherry picked from commit d0c6d50ff79d34cf19204b9ab0a0dc8b03fb6b15) --- src/GitTools.jl | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/GitTools.jl b/src/GitTools.jl index 562dada8ef..2569b87652 100644 --- a/src/GitTools.jl +++ b/src/GitTools.jl @@ -89,7 +89,7 @@ function checkout_tree_to_path(repo::LibGit2.GitRepo, tree::LibGit2.GitObject, p end end -function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kwargs...) +function clone(io::IO, url, source_path; header=nothing, credentials=nothing, isbare=false, kwargs...) url = String(url)::String source_path = String(source_path)::String @assert !isdir(source_path) || isempty(readdir(source_path)) @@ -103,7 +103,9 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw end try if use_cli_git() - cmd = `git clone --quiet $url $source_path` + args = ["--quiet", url, source_path] + isbare && pushfirst!(args, "--bare") + cmd = `git clone $args` try run(pipeline(cmd; stdout=devnull)) catch err @@ -122,7 +124,7 @@ function clone(io::IO, url, source_path; header=nothing, credentials=nothing, kw LibGit2.Callbacks() end mkpath(source_path) - return LibGit2.clone(url, source_path; callbacks, credentials, kwargs...) + return LibGit2.clone(url, source_path; callbacks, credentials, isbare, kwargs...) end catch err rm(source_path; force=true, recursive=true) @@ -151,7 +153,6 @@ function fetch(io::IO, repo::LibGit2.GitRepo, remoteurl=nothing; header=nothing, remoteurl = normalize_url(remoteurl) printpkgstyle(io, :Updating, header === nothing ? "git-repo `$remoteurl`" : header) bar = MiniProgressBar(header = "Fetching:", color = Base.info_color()) - fancyprint = can_fancyprint(io) callbacks = if fancyprint LibGit2.Callbacks( :transfer_progress => ( From ff55af2f173ebb06a4930de45e7a20553dcfbde8 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Fri, 20 Jun 2025 07:40:22 +0200 Subject: [PATCH 12/19] add update function to apps and fix a bug when adding an already installed app (#4263) Co-authored-by: KristofferC (cherry picked from commit e3d4561272fc029e9a5f940fe101ba4570fa875d) --- src/Apps/Apps.jl | 54 +++++++++++++++++++++++++--- src/REPLMode/command_declarations.jl | 15 ++++++++ 2 files changed, 64 insertions(+), 5 deletions(-) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index f6afef54f9..2cb20ab61b 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -111,7 +111,6 @@ function _resolve(manifest::Manifest, pkgname=nothing) # TODO: Julia path generate_shims_for_apps(pkg.name, pkg.apps, dirname(projectfile), joinpath(Sys.BINDIR, "julia")) end - write_manifest(manifest, app_manifest_file()) end @@ -145,10 +144,12 @@ function add(pkg::PackageSpec) new = Pkg.Operations.download_source(ctx, pkgs) end + # Run Pkg.build()? + + Base.rm(joinpath(app_env_folder(), pkg.name); force=true, recursive=true) sourcepath = source_path(ctx.env.manifest_file, pkg) project = get_project(sourcepath) # TODO: Wrong if package itself has a sourcepath? - entry = PackageEntry(;apps = project.apps, name = pkg.name, version = project.version, tree_hash = pkg.tree_hash, path = pkg.path, repo = pkg.repo, uuid=pkg.uuid) manifest.deps[pkg.uuid] = entry @@ -193,6 +194,50 @@ function develop(pkg::PackageSpec) @info "For package: $(pkg.name) installed apps: $(join(keys(project.apps), ","))" end + +update(pkgs_or_apps::String) = update([pkgs_or_apps]) +function update(pkgs_or_apps::Vector) + for pkg_or_app in pkgs_or_apps + if pkg_or_app isa String + pkg_or_app = PackageSpec(pkg_or_app) + end + update(pkg_or_app) + end +end + +function update(pkg::Union{PackageSpec, Nothing}=nothing) + ctx = app_context() + manifest = ctx.env.manifest + deps = Pkg.Operations.load_manifest_deps(manifest) + for dep in deps + info = manifest.deps[dep.uuid] + if pkg === nothing || info.name !== pkg.name + continue + end + Pkg.activate(joinpath(app_env_folder(), info.name)) do + # precompile only after updating all apps? + if pkg !== nothing + Pkg.update(pkg) + else + Pkg.update() + end + end + sourcepath = abspath(source_path(ctx.env.manifest_file, info)) + project = get_project(sourcepath) + # Get the tree hash from the project file + manifest_file = manifestfile_path(joinpath(app_env_folder(), info.name)) + manifest_app = Pkg.Types.read_manifest(manifest_file) + manifest_entry = manifest_app.deps[info.uuid] + + entry = PackageEntry(;apps = project.apps, name = manifest_entry.name, version = manifest_entry.version, tree_hash = manifest_entry.tree_hash, + path = manifest_entry.path, repo = manifest_entry.repo, uuid = manifest_entry.uuid) + + manifest.deps[dep.uuid] = entry + Pkg.Types.write_manifest(manifest, app_manifest_file()) + end + return +end + function status(pkgs_or_apps::Vector) if isempty(pkgs_or_apps) status() @@ -257,8 +302,7 @@ end function require_not_empty(pkgs, f::Symbol) - - if pkgs == nothing || isempty(pkgs) + if pkgs === nothing || isempty(pkgs) pkgerror("app $f requires at least one package") end end @@ -306,7 +350,7 @@ function rm(pkg_or_app::Union{PackageSpec, Nothing}=nothing) end end end - + # XXX: What happens if something fails above and we do not write out the updated manifest? Pkg.Types.write_manifest(manifest, app_manifest_file()) return end diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index a9d9cc3304..98153e5b48 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -647,6 +647,21 @@ pkg> app develop ~/mypackages/Example pkg> app develop --local Example ``` """ +], +PSA[:name => "update", + :short_name => "up", + :api => Apps.update, + :completions => :complete_installed_apps, + :should_splat => false, + :arg_count => 0 => Inf, + :arg_parser => parse_package, + :description => "update app", + :help => md""" + app update pkg + +Updates the apps for packages `pkg...` or apps `app...`. +``` +""", ], # app ] ] #command_declarations From 6de46f20c4893ea46287506bf7dff1f3ae436688 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Mon, 30 Jun 2025 23:21:11 +0200 Subject: [PATCH 13/19] Various app improvements (#4278) Co-authored-by: KristofferC (cherry picked from commit 109eaea66a0adb0ad8fa497e64913eadc2248ad1) --- src/Apps/Apps.jl | 91 +++++++++++++++++++++++++++++++++++++++++------- src/Types.jl | 1 - src/manifest.jl | 4 +-- src/project.jl | 2 +- 4 files changed, 81 insertions(+), 17 deletions(-) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 2cb20ab61b..770c872448 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -15,8 +15,41 @@ julia_bin_path() = joinpath(first(DEPOT_PATH), "bin") app_context() = Context(env=EnvCache(joinpath(app_env_folder(), "Project.toml"))) +function validate_app_name(name::AbstractString) + if isempty(name) + error("App name cannot be empty") + end + if !occursin(r"^[a-zA-Z][a-zA-Z0-9_-]*$", name) + error("App name must start with a letter and contain only letters, numbers, underscores, and hyphens") + end + if occursin(r"\.\.", name) || occursin(r"[/\\]", name) + error("App name cannot contain path traversal sequences or path separators") + end +end + +function validate_package_name(name::AbstractString) + if isempty(name) + error("Package name cannot be empty") + end + if !occursin(r"^[a-zA-Z][a-zA-Z0-9_]*$", name) + error("Package name must start with a letter and contain only letters, numbers, and underscores") + end +end + +function validate_submodule_name(name::Union{AbstractString,Nothing}) + if name !== nothing + if isempty(name) + error("Submodule name cannot be empty") + end + if !occursin(r"^[a-zA-Z][a-zA-Z0-9_]*$", name) + error("Submodule name must start with a letter and contain only letters, numbers, and underscores") + end + end +end + function rm_shim(name; kwargs...) + validate_app_name(name) Base.rm(joinpath(julia_bin_path(), name * (Sys.iswindows() ? ".bat" : "")); kwargs...) end @@ -38,6 +71,30 @@ function overwrite_file_if_different(file, content) end end +function check_apps_in_path(apps) + for app_name in keys(apps) + which_result = Sys.which(app_name) + if which_result === nothing + @warn """ + App '$app_name' was installed but is not available in PATH. + Consider adding '$(julia_bin_path())' to your PATH environment variable. + """ maxlog=1 + break # Only show warning once per installation + else + # Check for collisions + expected_path = joinpath(julia_bin_path(), app_name * (Sys.iswindows() ? ".bat" : "")) + if which_result != expected_path + @warn """ + App '$app_name' collision detected: + Expected: $expected_path + Found: $which_result + Another application with the same name exists in PATH. + """ + end + end + end +end + function get_max_version_register(pkg::PackageSpec, regs) max_v = nothing tree_hash = nothing @@ -157,6 +214,7 @@ function add(pkg::PackageSpec) precompile(pkg.name) @info "For package: $(pkg.name) installed apps $(join(keys(project.apps), ","))" + check_apps_in_path(project.apps) end function develop(pkg::Vector{PackageSpec}) @@ -192,6 +250,7 @@ function develop(pkg::PackageSpec) _resolve(manifest, pkg.name) precompile(pkg.name) @info "For package: $(pkg.name) installed apps: $(join(keys(project.apps), ","))" + check_apps_in_path(project.apps) end @@ -205,6 +264,7 @@ function update(pkgs_or_apps::Vector) end end +# XXX: Is updating an app ever different from rm-ing and adding it from scratch? function update(pkg::Union{PackageSpec, Nothing}=nothing) ctx = app_context() manifest = ctx.env.manifest @@ -385,7 +445,6 @@ const SHIM_VERSION = 1.0 const SHIM_HEADER = """$SHIM_COMMENT This file is generated by the Julia package manager. $SHIM_COMMENT Shim version: $SHIM_VERSION""" - function generate_shims_for_apps(pkgname, apps, env, julia) for (_, app) in apps generate_shim(pkgname, app, env, julia) @@ -393,13 +452,23 @@ function generate_shims_for_apps(pkgname, apps, env, julia) end function generate_shim(pkgname, app::AppInfo, env, julia) + validate_package_name(pkgname) + validate_app_name(app.name) + validate_submodule_name(app.submodule) + + module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" + filename = app.name * (Sys.iswindows() ? ".bat" : "") julia_bin_filename = joinpath(julia_bin_path(), filename) - mkpath(dirname(filename)) + mkpath(dirname(julia_bin_filename)) content = if Sys.iswindows() - windows_shim(pkgname, app, julia, env) + julia_escaped = "\"$(Base.shell_escape_wincmd(julia))\"" + module_spec_escaped = "\"$(Base.shell_escape_wincmd(module_spec))\"" + windows_shim(julia_escaped, module_spec_escaped, env) else - bash_shim(pkgname, app, julia, env) + julia_escaped = Base.shell_escape(julia) + module_spec_escaped = Base.shell_escape(module_spec) + bash_shim(julia_escaped, module_spec_escaped, env) end overwrite_file_if_different(julia_bin_filename, content) if Sys.isunix() @@ -408,8 +477,7 @@ function generate_shim(pkgname, app::AppInfo, env, julia) end -function bash_shim(pkgname, app::AppInfo, julia::String, env) - module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" +function bash_shim(julia_escaped::String, module_spec_escaped::String, env) return """ #!/usr/bin/env bash @@ -417,15 +485,14 @@ function bash_shim(pkgname, app::AppInfo, julia::String, env) export JULIA_LOAD_PATH=$(repr(env)) export JULIA_DEPOT_PATH=$(repr(join(DEPOT_PATH, ':'))) - exec $julia \\ + exec $julia_escaped \\ --startup-file=no \\ - -m $(module_spec) \\ + -m $module_spec_escaped \\ "\$@" """ end -function windows_shim(pkgname, app::AppInfo, julia::String, env) - module_spec = app.submodule === nothing ? pkgname : "$(pkgname).$(app.submodule)" +function windows_shim(julia_escaped::String, module_spec_escaped::String, env) return """ @echo off @@ -435,9 +502,9 @@ function windows_shim(pkgname, app::AppInfo, julia::String, env) set JULIA_LOAD_PATH=$env set JULIA_DEPOT_PATH=$(join(DEPOT_PATH, ';')) - $julia ^ + $julia_escaped ^ --startup-file=no ^ - -m $(module_spec) ^ + -m $module_spec_escaped ^ %* """ end diff --git a/src/Types.jl b/src/Types.jl index 1ead221dc2..eb6ca21eff 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -241,7 +241,6 @@ Base.hash(t::Compat, h::UInt) = hash(t.val, h) struct AppInfo name::String julia_command::Union{String, Nothing} - julia_version::Union{VersionNumber, Nothing} submodule::Union{String, Nothing} other::Dict{String,Any} end diff --git a/src/manifest.jl b/src/manifest.jl index 5eb1812106..92da17849c 100644 --- a/src/manifest.jl +++ b/src/manifest.jl @@ -89,7 +89,6 @@ function read_apps(apps::Dict) submodule = get(app, "submodule", nothing) appinfo = AppInfo(appname::String, app["julia_command"]::String, - VersionNumber(app["julia_version"]::String), submodule, app) appinfos[appinfo.name] = appinfo @@ -335,8 +334,7 @@ function destructure(manifest::Manifest)::Dict new_entry["apps"] = Dict{String,Any}() for (appname, appinfo) in entry.apps julia_command = @something appinfo.julia_command joinpath(Sys.BINDIR, "julia" * (Sys.iswindows() ? ".exe" : "")) - julia_version = @something appinfo.julia_version VERSION - app_dict = Dict{String,Any}("julia_command" => julia_command, "julia_version" => julia_version) + app_dict = Dict{String,Any}("julia_command" => julia_command) if appinfo.submodule !== nothing app_dict["submodule"] = appinfo.submodule end diff --git a/src/project.jl b/src/project.jl index 807bc61e98..c82aab7278 100644 --- a/src/project.jl +++ b/src/project.jl @@ -83,7 +83,7 @@ function read_project_apps(raw::Dict{String,Any}, project::Project) Expected value for app `$name` to be a dictionary. """) submodule = get(info, "submodule", nothing) - appinfos[name] = AppInfo(name, nothing, nothing, submodule, other) + appinfos[name] = AppInfo(name, nothing, submodule, other) end return appinfos end From 7e7d6d169661fef1d561f7ebf95ab2275f9a1e48 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 1 Jul 2025 15:59:36 +0200 Subject: [PATCH 14/19] Various improvements to docs and docstrings (#4279) (cherry picked from commit aaff8f2f98b64d1d33a2706aa0ee1ec5a5047a8f) --- docs/make.jl | 1 + docs/src/api.md | 2 +- docs/src/apps.md | 24 ++++---- docs/src/artifacts.md | 2 +- docs/src/compatibility.md | 4 +- docs/src/creating-packages.md | 11 +++- docs/src/environments.md | 20 +++---- docs/src/getting-started.md | 36 ++++++------ docs/src/glossary.md | 4 +- docs/src/managing-packages.md | 82 ++++++++++++++-------------- docs/src/registries.md | 2 +- docs/src/toml-files.md | 2 +- src/Pkg.jl | 28 +++++++++- src/REPLMode/command_declarations.jl | 39 +++++++++---- 14 files changed, 154 insertions(+), 103 deletions(-) diff --git a/docs/make.jl b/docs/make.jl index be6905de5a..976c1e4a21 100644 --- a/docs/make.jl +++ b/docs/make.jl @@ -35,6 +35,7 @@ makedocs( "managing-packages.md", "environments.md", "creating-packages.md", + "apps.md", "compatibility.md", "registries.md", "artifacts.md", diff --git a/docs/src/api.md b/docs/src/api.md index 61979453b9..dc9e9e1794 100644 --- a/docs/src/api.md +++ b/docs/src/api.md @@ -1,4 +1,4 @@ -# [**12.** API Reference](@id API-Reference) +# [**13.** API Reference](@id API-Reference) This section describes the functional API for interacting with Pkg.jl. It is recommended to use the functional API, rather than the Pkg REPL mode, diff --git a/docs/src/apps.md b/docs/src/apps.md index 083d68f0e1..00b12cada9 100644 --- a/docs/src/apps.md +++ b/docs/src/apps.md @@ -1,4 +1,4 @@ -# [**?.** Apps](@id Apps) +# [**6.** Apps](@id Apps) !!! note The app support in Pkg is currently considered experimental and some functionality and API may change. @@ -8,7 +8,7 @@ - The path to the julia executable used is the same as the one used to install the app. If this julia installation gets removed, you might need to reinstall the app. -Apps are Julia packages that are intended to be run as a "standalone programs" (by e.g. typing the name of the app in the terminal possibly together with some arguments or flags/options). +Apps are Julia packages that are intended to be run as "standalone programs" (by e.g. typing the name of the app in the terminal possibly together with some arguments or flags/options). This is in contrast to most Julia packages that are used as "libraries" and are loaded by other files or in the Julia REPL. ## Creating a Julia app @@ -44,6 +44,15 @@ reverse = {} ``` The empty table `{}` is to allow for giving metadata about the app but it is currently unused. +After installing this app one could run: + +``` +$ reverse some input string + emos tupni gnirts +``` + +directly in the terminal. + ## Multiple Apps per Package A single package can define multiple apps by using submodules. Each app can have its own entry point in a different submodule of the package. @@ -86,15 +95,6 @@ This will create two executables: - `main-app` that runs `julia -m MyMultiApp` - `cli-app` that runs `julia -m MyMultiApp.CLI` -After installing this app one could run: - -``` -$ reverse some input string -emos tupni gnirts -``` - -directly in the terminal. - ## Installing Julia apps -The installation of Julia apps are similar to installing julia libraries but instead of using e.g. `Pkg.add` or `pkg> add` one uses `Pkg.Apps.add` or `pkg> app add` (`develop` is also available). +The installation of Julia apps is similar to [installing Julia libraries](@ref Managing-Packages) but instead of using e.g. `Pkg.add` or `pkg> add` one uses `Pkg.Apps.add` or `pkg> app add` (`develop` is also available). diff --git a/docs/src/artifacts.md b/docs/src/artifacts.md index 66a55f99f5..d804dfeb10 100644 --- a/docs/src/artifacts.md +++ b/docs/src/artifacts.md @@ -1,4 +1,4 @@ -# [**8.** Artifacts](@id Artifacts) +# [**9.** Artifacts](@id Artifacts) `Pkg` can install and manage containers of data that are not Julia packages. These containers can contain platform-specific binaries, datasets, text, or any other kind of data that would be convenient to place within an immutable, life-cycled datastore. These containers, (called "Artifacts") can be created locally, hosted anywhere, and automatically downloaded and unpacked upon installation of your Julia package. diff --git a/docs/src/compatibility.md b/docs/src/compatibility.md index bc1c58e3e9..e9b7527ea1 100644 --- a/docs/src/compatibility.md +++ b/docs/src/compatibility.md @@ -1,4 +1,4 @@ -# [**6.** Compatibility](@id Compatibility) +# [**7.** Compatibility](@id Compatibility) Compatibility refers to the ability to restrict the versions of the dependencies that your project is compatible with. If the compatibility for a dependency is not given, the project is assumed to be compatible with all versions of that dependency. @@ -164,7 +164,7 @@ PkgA = "0.2 - 0" # 0.2.0 - 0.*.* = [0.2.0, 1.0.0) ``` -## Fixing conflicts +## [Fixing conflicts](@id Fixing-conflicts) Version conflicts were introduced previously with an [example](@ref conflicts) of a conflict arising in a package `D` used by two other packages, `B` and `C`. diff --git a/docs/src/creating-packages.md b/docs/src/creating-packages.md index 7bb72c2e91..ff0b67ee2e 100644 --- a/docs/src/creating-packages.md +++ b/docs/src/creating-packages.md @@ -11,7 +11,7 @@ To generate the bare minimum files for a new package, use `pkg> generate`. ```julia-repl -(@v1.8) pkg> generate HelloWorld +(@v1.10) pkg> generate HelloWorld ``` This creates a new project `HelloWorld` in a subdirectory by the same name, with the following files (visualized with the external [`tree` command](https://linux.die.net/man/1/tree)): @@ -118,7 +118,7 @@ describe about public symbols. A public symbol is a symbol that is exported from package with the `export` keyword or marked as public with the `public` keyword. When you change the behavior of something that was previously public so that the new version no longer conforms to the specifications provided in the old version, you should -adjust your package version number according to [Julia's variant on SemVer](#Version-specifier-format). +adjust your package version number according to [Julia's variant on SemVer](@ref Version-specifier-format). If you would like to include a symbol in your public API without exporting it into the global namespace of folks who call `using YourPackage`, you should mark that symbol as public with `public that_symbol`. Symbols marked as public with the `public` keyword are @@ -649,3 +649,10 @@ To support the various use cases in the Julia package ecosystem, the Pkg develop * [`Preferences.jl`](https://github.com/JuliaPackaging/Preferences.jl) allows packages to read and write preferences to the top-level `Project.toml`. These preferences can be read at runtime or compile-time, to enable or disable different aspects of package behavior. Packages previously would write out files to their own package directories to record options set by the user or environment, but this is highly discouraged now that `Preferences` is available. + +## See Also + +- [Managing Packages](@ref Managing-Packages) - Learn how to add, update, and manage package dependencies +- [Working with Environments](@ref Working-with-Environments) - Understand environments and reproducible development +- [Compatibility](@ref Compatibility) - Specify version constraints for dependencies +- [API Reference](@ref) - Functional API for non-interactive package management diff --git a/docs/src/environments.md b/docs/src/environments.md index fd16427e10..d22ef8e58b 100644 --- a/docs/src/environments.md +++ b/docs/src/environments.md @@ -10,7 +10,7 @@ It should be pointed out that when two projects use the same package at the same In order to create a new project, create a directory for it and then activate that directory to make it the "active project", which package operations manipulate: ```julia-repl -(@v1.9) pkg> activate MyProject +(@v1.10) pkg> activate MyProject Activating new environment at `~/MyProject/Project.toml` (MyProject) pkg> st @@ -28,7 +28,7 @@ false Installed Example ─ v0.5.3 Updating `~/MyProject/Project.toml` [7876af07] + Example v0.5.3 - Updating `~~/MyProject/Manifest.toml` + Updating `~/MyProject/Manifest.toml` [7876af07] + Example v0.5.3 Precompiling environment... 1 dependency successfully precompiled in 2 seconds @@ -45,7 +45,7 @@ Example = "7876af07-990d-54b4-ab0e-23690620f79a" julia> print(read(joinpath("MyProject", "Manifest.toml"), String)) # This file is machine-generated - editing it directly is not advised -julia_version = "1.9.4" +julia_version = "1.10.0" manifest_format = "2.0" project_hash = "2ca1c6c58cb30e79e021fb54e5626c96d05d5fdc" @@ -66,7 +66,7 @@ shell> git clone https://github.com/JuliaLang/Example.jl.git Cloning into 'Example.jl'... ... -(@v1.12) pkg> activate Example.jl +(@v1.10) pkg> activate Example.jl Activating project at `~/Example.jl` (Example) pkg> instantiate @@ -82,7 +82,7 @@ If you only have a `Project.toml`, a `Manifest.toml` must be generated by "resol If you already have a resolved `Manifest.toml`, then you will still need to ensure that the packages are installed and with the correct versions. Again `instantiate` does this for you. -In short, `instantiate` is your friend to make sure an environment is ready to use. If there's nothing to do, `instantiate` does nothing. +In short, [`instantiate`](@ref Pkg.instantiate) is your friend to make sure an environment is ready to use. If there's nothing to do, `instantiate` does nothing. !!! note "Specifying project on startup" Instead of using `activate` from within Julia, you can specify the project on startup using @@ -103,7 +103,7 @@ also want a scratch space to try out a new package, or a sandbox to resolve vers between several incompatible packages. ```julia-repl -(@v1.9) pkg> activate --temp # requires Julia 1.5 or later +(@v1.10) pkg> activate --temp # requires Julia 1.5 or later Activating new environment at `/var/folders/34/km3mmt5930gc4pzq1d08jvjw0000gn/T/jl_a31egx/Project.toml` (jl_a31egx) pkg> add Example @@ -121,14 +121,14 @@ A "shared" environment is simply an environment that exists in `~/.julia/environ therefore a shared environment: ```julia-repl -(@v1.9) pkg> st +(@v1.10) pkg> st Status `~/.julia/environments/v1.9/Project.toml` ``` Shared environments can be activated with the `--shared` flag to `activate`: ```julia-repl -(@v1.9) pkg> activate --shared mysharedenv +(@v1.10) pkg> activate --shared mysharedenv Activating project at `~/.julia/environments/mysharedenv` (@mysharedenv) pkg> @@ -151,7 +151,7 @@ or using Pkg's precompile option, which can precompile the entire environment, o which can be significantly faster than the code-load route above. ```julia-repl -(@v1.9) pkg> precompile +(@v1.10) pkg> precompile Precompiling environment... 23 dependencies successfully precompiled in 36 seconds ``` @@ -165,7 +165,7 @@ By default, any package that is added to a project or updated in a Pkg action wi with its dependencies. ```julia-repl -(@v1.9) pkg> add Images +(@v1.10) pkg> add Images Resolving package versions... Updating `~/.julia/environments/v1.9/Project.toml` [916415d5] + Images v0.25.2 diff --git a/docs/src/getting-started.md b/docs/src/getting-started.md index 58693bc583..93acb7c613 100644 --- a/docs/src/getting-started.md +++ b/docs/src/getting-started.md @@ -22,18 +22,18 @@ To get back to the Julia REPL, press `Ctrl+C` or backspace (when the REPL cursor Upon entering the Pkg REPL, you should see the following prompt: ```julia-repl -(@v1.9) pkg> +(@v1.10) pkg> ``` To add a package, use `add`: ```julia-repl -(@v1.9) pkg> add Example +(@v1.10) pkg> add Example Resolving package versions... Installed Example ─ v0.5.3 - Updating `~/.julia/environments/v1.9/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] + Example v0.5.3 - Updating `~/.julia/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [7876af07] + Example v0.5.3 ``` @@ -49,14 +49,14 @@ julia> Example.hello("friend") We can also specify multiple packages at once to install: ```julia-repl -(@v1.9) pkg> add JSON StaticArrays +(@v1.10) pkg> add JSON StaticArrays ``` The `status` command (or the shorter `st` command) can be used to see installed packages. ```julia-repl -(@v1.9) pkg> st -Status `~/.julia/environments/v1.6/Project.toml` +(@v1.10) pkg> st +Status `~/.julia/environments/v1.10/Project.toml` [7876af07] Example v0.5.3 [682c06a0] JSON v0.21.3 [90137ffa] StaticArrays v1.5.9 @@ -68,13 +68,13 @@ Status `~/.julia/environments/v1.6/Project.toml` To remove packages, use `rm` (or `remove`): ```julia-repl -(@v1.9) pkg> rm JSON StaticArrays +(@v1.10) pkg> rm JSON StaticArrays ``` Use `up` (or `update`) to update the installed packages ```julia-repl -(@v1.9) pkg> up +(@v1.10) pkg> up ``` If you have been following this guide it is likely that the packages installed are at the latest version @@ -82,13 +82,13 @@ so `up` will not do anything. Below we show the status output in the case where an old version of the Example package and then upgrade it: ```julia-repl -(@v1.9) pkg> st -Status `~/.julia/environments/v1.9/Project.toml` +(@v1.10) pkg> st +Status `~/.julia/environments/v1.10/Project.toml` ⌃ [7876af07] Example v0.5.1 Info Packages marked with ⌃ have new versions available and may be upgradable. -(@v1.9) pkg> up - Updating `~/.julia/environments/v1.9/Project.toml` +(@v1.10) pkg> up + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ↑ Example v0.5.1 ⇒ v0.5.3 ``` @@ -110,7 +110,7 @@ Let's set up a new environment so we may experiment. To set the active environment, use `activate`: ```julia-repl -(@v1.9) pkg> activate tutorial +(@v1.10) pkg> activate tutorial [ Info: activating new environment at `~/tutorial/Project.toml`. ``` @@ -166,16 +166,16 @@ For more information about environments, see the [Working with Environments](@re If you are ever stuck, you can ask `Pkg` for help: ```julia-repl -(@v1.9) pkg> ? +(@v1.10) pkg> ? ``` You should see a list of available commands along with short descriptions. You can ask for more detailed help by specifying a command: ```julia-repl -(@v1.9) pkg> ?develop +(@v1.10) pkg> ?develop ``` This guide should help you get started with `Pkg`. -`Pkg` has much more to offer in terms of powerful package management, -read the full manual to learn more! +`Pkg` has much more to offer in terms of powerful package management. +For more advanced topics, see [Managing Packages](@ref Managing-Packages), [Working with Environments](@ref Working-with-Environments), and [Creating Packages](@ref creating-packages-tutorial). diff --git a/docs/src/glossary.md b/docs/src/glossary.md index 60e0546039..54c00aa8ea 100644 --- a/docs/src/glossary.md +++ b/docs/src/glossary.md @@ -1,4 +1,4 @@ -# [**9.** Glossary](@id Glossary) +# [**10.** Glossary](@id Glossary) **Project:** a source tree with a standard layout, including a `src` directory for the main body of Julia code, a `test` directory for testing the project, @@ -46,7 +46,7 @@ since that could conflict with the configuration of the main application. **Environment:** the combination of the top-level name map provided by a project file combined with the dependency graph and map from packages to their entry points -provided by a manifest file. For more detail see the manual section on code loading. +provided by a manifest file. For more detail see the [manual section on code loading](https://docs.julialang.org/en/v1/manual/code-loading/). - **Explicit environment:** an environment in the form of an explicit project file and an optional corresponding manifest file together in a directory. If the diff --git a/docs/src/managing-packages.md b/docs/src/managing-packages.md index b5889221cf..ec6cf34be8 100644 --- a/docs/src/managing-packages.md +++ b/docs/src/managing-packages.md @@ -10,14 +10,14 @@ The most frequently used is `add` and its usage is described first. In the Pkg REPL, packages can be added with the `add` command followed by the name of the package, for example: ```julia-repl -(@v1.8) pkg> add JSON +(@v1.10) pkg> add JSON Installing known registries into `~/` Resolving package versions... Installed Parsers ─ v2.4.0 Installed JSON ──── v0.21.3 - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [682c06a0] + JSON v0.21.3 - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [682c06a0] + JSON v0.21.3 [69de0a69] + Parsers v2.4.0 [ade2ca70] + Dates @@ -40,16 +40,16 @@ It is possible to add multiple packages in one command as `pkg> add A B C`. The status output contains the packages you have added yourself, in this case, `JSON`: ```julia-repl -(@v1.11) pkg> st - Status `~/.julia/environments/v1.8/Project.toml` +(@v1.10) pkg> st + Status `~/.julia/environments/v1.10/Project.toml` [682c06a0] JSON v0.21.3 ``` The manifest status shows all the packages in the environment, including recursive dependencies: ```julia-repl -(@v1.11) pkg> st -m -Status `~/environments/v1.9/Manifest.toml` +(@v1.10) pkg> st -m +Status `~/.julia/environments/v1.10/Manifest.toml` [682c06a0] JSON v0.21.3 [69de0a69] Parsers v2.4.0 [ade2ca70] Dates @@ -64,18 +64,18 @@ To specify that you want a particular version (or set of versions) of a package, to require any patch release of the v0.21 series of JSON after v0.21.4, call `compat JSON 0.21.4`: ```julia-repl -(@1.11) pkg> compat JSON 0.21.4 +(@v1.10) pkg> compat JSON 0.21.4 Compat entry set: JSON = "0.21.4" Resolve checking for compliance with the new compat rules... Error empty intersection between JSON@0.21.3 and project compatibility 0.21.4 - 0.21 Suggestion Call `update` to attempt to meet the compatibility requirements. -(@1.11) pkg> update +(@v1.10) pkg> update Updating registry at `~/.julia/registries/General.toml` - Updating `~/.julia/environments/1.11/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [682c06a0] ↑ JSON v0.21.3 ⇒ v0.21.4 - Updating `~/.julia/environments/1.11/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [682c06a0] ↑ JSON v0.21.3 ⇒ v0.21.4 ``` @@ -96,11 +96,11 @@ julia> JSON.json(Dict("foo" => [1, "bar"])) |> print A specific version of a package can be installed by appending a version after a `@` symbol to the package name: ```julia-repl -(@v1.8) pkg> add JSON@0.21.1 +(@v1.10) pkg> add JSON@0.21.1 Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` ⌃ [682c06a0] + JSON v0.21.1 - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` ⌃ [682c06a0] + JSON v0.21.1 ⌅ [69de0a69] + Parsers v1.1.2 [ade2ca70] + Dates @@ -118,12 +118,12 @@ If a branch (or a certain commit) of `Example` has a hotfix that is not yet incl we can explicitly track that branch (or commit) by appending `#branchname` (or `#commitSHA1`) to the package name: ```julia-repl -(@v1.8) pkg> add Example#master +(@v1.10) pkg> add Example#master Cloning git-repo `https://github.com/JuliaLang/Example.jl.git` Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] + Example v0.5.4 `https://github.com/JuliaLang/Example.jl.git#master` - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [7876af07] + Example v0.5.4 `https://github.com/JuliaLang/Example.jl.git#master` ``` @@ -139,12 +139,12 @@ When updating packages, updates are pulled from that branch. To go back to tracking the registry version of `Example`, the command `free` is used: ```julia-repl -(@v1.8) pkg> free Example +(@v1.10) pkg> free Example Resolving package versions... Installed Example ─ v0.5.3 - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ~ Example v0.5.4 `https://github.com/JuliaLang/Example.jl.git#master` ⇒ v0.5.3 - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [7876af07] ~ Example v0.5.4 `https://github.com/JuliaLang/Example.jl.git#master` ⇒ v0.5.3 ``` @@ -153,12 +153,12 @@ To go back to tracking the registry version of `Example`, the command `free` is If a package is not in a registry, it can be added by specifying a URL to the Git repository: ```julia-repl -(@v1.8) pkg> add https://github.com/fredrikekre/ImportMacros.jl +(@v1.10) pkg> add https://github.com/fredrikekre/ImportMacros.jl Cloning git-repo `https://github.com/fredrikekre/ImportMacros.jl` Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [92a963f6] + ImportMacros v1.0.0 `https://github.com/fredrikekre/ImportMacros.jl#master` - Updating `~/environments/v1.9/Manifest.toml` + Updating `~/.julia/environments/v1.10/Manifest.toml` [92a963f6] + ImportMacros v1.0.0 `https://github.com/fredrikekre/ImportMacros.jl#master` ``` @@ -167,7 +167,7 @@ For unregistered packages, we could have given a branch name (or commit SHA1) to If you want to add a package using the SSH-based `git` protocol, you have to use quotes because the URL contains a `@`. For example, ```julia-repl -(@v1.8) pkg> add "git@github.com:fredrikekre/ImportMacros.jl.git" +(@v1.10) pkg> add "git@github.com:fredrikekre/ImportMacros.jl.git" Cloning git-repo `git@github.com:fredrikekre/ImportMacros.jl.git` Updating registry at `~/.julia/registries/General` Resolving package versions... @@ -188,7 +188,7 @@ repository: pkg> add https://github.com/timholy/SnoopCompile.jl.git:SnoopCompileCore Cloning git-repo `https://github.com/timholy/SnoopCompile.jl.git` Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [e2b509da] + SnoopCompileCore v2.9.0 `https://github.com/timholy/SnoopCompile.jl.git:SnoopCompileCore#master` Updating `~/.julia/environments/v1.8/Manifest.toml` [e2b509da] + SnoopCompileCore v2.9.0 `https://github.com/timholy/SnoopCompile.jl.git:SnoopCompileCore#master` @@ -214,15 +214,15 @@ from that local repo are pulled when packages are updated. By only using `add` your environment always has a "reproducible state", in other words, as long as the repositories and registries used are still accessible it is possible to retrieve the exact state of all the dependencies in the environment. This has the advantage that you can send your environment (`Project.toml` and `Manifest.toml`) to someone else and they can [`Pkg.instantiate`](@ref) that environment in the same state as you had it locally. -However, when you are developing a package, it is more convenient to load packages at their current state at some path. For this reason, the `dev` command exists. +However, when you are [developing a package](@ref developing), it is more convenient to load packages at their current state at some path. For this reason, the `dev` command exists. Let's try to `dev` a registered package: ```julia-repl -(@v1.8) pkg> dev Example +(@v1.10) pkg> dev Example Updating git-repo `https://github.com/JuliaLang/Example.jl.git` Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] + Example v0.5.4 `~/.julia/dev/Example` Updating `~/.julia/environments/v1.8/Manifest.toml` [7876af07] + Example v0.5.4 `~/.julia/dev/Example` @@ -263,9 +263,9 @@ julia> Example.plusone(1) To stop tracking a path and use the registered version again, use `free`: ```julia-repl -(@v1.8) pkg> free Example +(@v1.10) pkg> free Example Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ~ Example v0.5.4 `~/.julia/dev/Example` ⇒ v0.5.3 Updating `~/.julia/environments/v1.8/Manifest.toml` [7876af07] ~ Example v0.5.4 `~/.julia/dev/Example` ⇒ v0.5.3 @@ -300,29 +300,29 @@ When new versions of packages are released, it is a good idea to update. Simply to the latest compatible version. Sometimes this is not what you want. You can specify a subset of the dependencies to upgrade by giving them as arguments to `up`, e.g: ```julia-repl -(@v1.8) pkg> up Example +(@v1.10) pkg> up Example ``` This will only allow Example do upgrade. If you also want to allow dependencies of Example to upgrade (with the exception of packages that are in the project) you can pass the `--preserve=direct` flag. ```julia-repl -(@v1.8) pkg> up --preserve=direct Example +(@v1.10) pkg> up --preserve=direct Example ``` And if you also want to allow dependencies of Example that are also in the project to upgrade, you can use `--preserve=none`: ```julia-repl -(@v1.8) pkg> up --preserve=none Example +(@v1.10) pkg> up --preserve=none Example ``` ## Pinning a package A pinned package will never be updated. A package can be pinned using `pin`, for example: ```julia-repl -(@v1.8) pkg> pin Example +(@v1.10) pkg> pin Example Resolving package versions... - Updating `~/.julia/environments/v1.8/Project.toml` + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ~ Example v0.5.3 ⇒ v0.5.3 ⚲ Updating `~/.julia/environments/v1.8/Manifest.toml` [7876af07] ~ Example v0.5.3 ⇒ v0.5.3 ⚲ @@ -331,8 +331,8 @@ A pinned package will never be updated. A package can be pinned using `pin`, for Note the pin symbol `⚲` showing that the package is pinned. Removing the pin is done using `free` ```julia-repl -(@v1.8) pkg> free Example - Updating `~/.julia/environments/v1.8/Project.toml` +(@v1.10) pkg> free Example + Updating `~/.julia/environments/v1.10/Project.toml` [7876af07] ~ Example v0.5.3 ⚲ ⇒ v0.5.3 Updating `~/.julia/environments/v1.8/Manifest.toml` [7876af07] ~ Example v0.5.3 ⚲ ⇒ v0.5.3 @@ -343,7 +343,7 @@ Note the pin symbol `⚲` showing that the package is pinned. Removing the pin i The tests for a package can be run using `test` command: ```julia-repl -(@v1.8) pkg> test Example +(@v1.10) pkg> test Example ... Testing Example Testing Example tests passed @@ -356,7 +356,7 @@ The output of the build process is directed to a file. To explicitly run the build step for a package, the `build` command is used: ```julia-repl -(@v1.8) pkg> build IJulia +(@v1.10) pkg> build IJulia Building Conda ─→ `~/.julia/scratchspaces/44cfe95a-1eb2-52ea-b672-e2afdf69b78f/6e47d11ea2776bc5627421d59cdcc1296c058071/build.log` Building IJulia → `~/.julia/scratchspaces/44cfe95a-1eb2-52ea-b672-e2afdf69b78f/98ab633acb0fe071b671f6c1785c46cd70bb86bd/build.log` @@ -486,7 +486,7 @@ To fix such errors, you have a number of options: - remove either `A` or `B` from your environment. Perhaps `B` is left over from something you were previously working on, and you don't need it anymore. If you don't need `A` and `B` at the same time, this is the easiest way to fix the problem. - try reporting your conflict. In this case, we were able to deduce that `B` requires an outdated version of `D`. You could thus report an issue in the development repository of `B.jl` asking for an updated version. - try fixing the problem yourself. - This becomes easier once you understand `Project.toml` files and how they declare their compatibility requirements. We'll return to this example in [Fixing conflicts](@ref). + This becomes easier once you understand `Project.toml` files and how they declare their compatibility requirements. We'll return to this example in [Fixing conflicts](@ref Fixing-conflicts). ## Garbage collecting old, unused packages @@ -502,7 +502,7 @@ If you are short on disk space and want to clean out as many unused packages and To run a typical garbage collection with default arguments, simply use the `gc` command at the `pkg>` REPL: ```julia-repl -(@v1.8) pkg> gc +(@v1.10) pkg> gc Active manifests at: `~/BinaryProvider/Manifest.toml` ... diff --git a/docs/src/registries.md b/docs/src/registries.md index 7c50727204..cada0bdadf 100644 --- a/docs/src/registries.md +++ b/docs/src/registries.md @@ -1,4 +1,4 @@ -# **7.** Registries +# **8.** Registries Registries contain information about packages, such as available releases and dependencies, and where they can be downloaded. diff --git a/docs/src/toml-files.md b/docs/src/toml-files.md index 262c1b5767..1d862a1cd4 100644 --- a/docs/src/toml-files.md +++ b/docs/src/toml-files.md @@ -1,4 +1,4 @@ -# [**10.** `Project.toml` and `Manifest.toml`](@id Project-and-Manifest) +# [**11.** `Project.toml` and `Manifest.toml`](@id Project-and-Manifest) Two files that are central to Pkg are `Project.toml` and `Manifest.toml`. `Project.toml` and `Manifest.toml` are written in [TOML](https://github.com/toml-lang/toml) (hence the diff --git a/src/Pkg.jl b/src/Pkg.jl index 8c7837d10d..f3d58d2b4a 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -371,8 +371,13 @@ To get updates from the origin path or remote repository the package must first # Examples ```julia +# Pin a package to its current version Pkg.pin("Example") + +# Pin a package to a specific version Pkg.pin(name="Example", version="0.3.1") + +# Pin all packages in the project Pkg.pin(all_pkgs = true) ``` """ @@ -391,7 +396,13 @@ To free all dependencies set `all_pkgs=true`. # Examples ```julia +# Free a single package (remove pin or stop tracking path) Pkg.free("Package") + +# Free multiple packages +Pkg.free(["PackageA", "PackageB"]) + +# Free all packages in the project Pkg.free(all_pkgs = true) ``` @@ -702,7 +713,17 @@ Other choices for `protocol` are `"https"` or `"git"`. ```julia-repl julia> Pkg.setprotocol!(domain = "github.com", protocol = "ssh") +# Use HTTPS for GitHub (default, good for most users) +julia> Pkg.setprotocol!(domain = "github.com", protocol = "https") + +# Reset to default (let package developer decide) +julia> Pkg.setprotocol!(domain = "github.com", protocol = nothing) + +# Set protocol for custom domain without specifying protocol julia> Pkg.setprotocol!(domain = "gitlab.mycompany.com") + +# Use Git protocol for a custom domain +julia> Pkg.setprotocol!(domain = "gitlab.mycompany.com", protocol = "git") ``` """ const setprotocol! = API.setprotocol! @@ -777,8 +798,11 @@ If the manifest doesn't have the project hash recorded, or if there is no manife This function can be used in tests to verify that the manifest is synchronized with the project file: - using Pkg, Test, Package - @test Pkg.is_manifest_current(pkgdir(Package)) +```julia +using Pkg, Test +@test Pkg.is_manifest_current(pwd()) # Check current project +@test Pkg.is_manifest_current("/path/to/project") # Check specific project +``` """ const is_manifest_current = API.is_manifest_current diff --git a/src/REPLMode/command_declarations.jl b/src/REPLMode/command_declarations.jl index 98153e5b48..9e016c3f76 100644 --- a/src/REPLMode/command_declarations.jl +++ b/src/REPLMode/command_declarations.jl @@ -16,7 +16,7 @@ PSA[:name => "test", test [--coverage] [pkg[=uuid]] ... Run the tests for package `pkg`, or for the current project (which thus needs to be -a package) if `pkg` is ommitted. This is done by running the file `test/runtests.jl` +a package) if `pkg` is omitted. This is done by running the file `test/runtests.jl` in the package directory. The option `--coverage` can be used to run the tests with coverage enabled. The `startup.jl` file is disabled during testing unless julia is started with `--startup-file=yes`. @@ -592,7 +592,10 @@ pkg> registry status :completions => :complete_installed_apps, :description => "show status of apps", :help => md""" - show status of apps + app status [pkg[=uuid]] ... + +Show the status of installed apps. If packages are specified, only show +apps for those packages. """ ], PSA[:name => "add", @@ -603,9 +606,15 @@ PSA[:name => "add", :completions => :complete_add_dev, :description => "add app", :help => md""" - app add pkg + app add pkg[=uuid] ... + +Add apps provided by packages `pkg...`. This will make the apps available +as executables in `~/.julia/bin` (which should be added to PATH). -Adds the apps for packages `pkg...` or apps `app...`. +**Examples** +``` +pkg> app add Example +pkg> app add Example@0.5.0 ``` """, ], @@ -616,12 +625,17 @@ PSA[:name => "remove", :arg_count => 0 => Inf, :arg_parser => parse_package, :completions => :complete_installed_apps, - :description => "remove packages from project or manifest", + :description => "remove apps", :help => md""" - app [rm|remove] pkg ... - app [rm|remove] app ... + app [rm|remove] pkg[=uuid] ... + +Remove apps provided by packages `pkg...`. This will remove the executables +from `~/.julia/bin`. - Remove the apps for package `pkg`. +**Examples** +``` +pkg> app rm Example +``` """ ], PSA[:name => "develop", @@ -657,9 +671,14 @@ PSA[:name => "update", :arg_parser => parse_package, :description => "update app", :help => md""" - app update pkg + app [up|update] [pkg[=uuid]] ... -Updates the apps for packages `pkg...` or apps `app...`. +Update apps for packages `pkg...`. If no packages are specified, all apps will be updated. + +**Examples** +``` +pkg> app update +pkg> app update Example ``` """, ], # app From 900125ed99291a3991108d3f4b6d26c86c9126f2 Mon Sep 17 00:00:00 2001 From: Timothy Date: Wed, 2 Jul 2025 15:58:02 +0800 Subject: [PATCH 15/19] Switch to more portable shell shebang (#4162) Co-authored-by: Kristoffer Carlsson (cherry picked from commit 175a1ffb636e2591bb2b4361251cd46990875ee6) --- src/Apps/Apps.jl | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/Apps/Apps.jl b/src/Apps/Apps.jl index 770c872448..c9e4260415 100644 --- a/src/Apps/Apps.jl +++ b/src/Apps/Apps.jl @@ -468,7 +468,7 @@ function generate_shim(pkgname, app::AppInfo, env, julia) else julia_escaped = Base.shell_escape(julia) module_spec_escaped = Base.shell_escape(module_spec) - bash_shim(julia_escaped, module_spec_escaped, env) + shell_shim(julia_escaped, module_spec_escaped, env) end overwrite_file_if_different(julia_bin_filename, content) if Sys.isunix() @@ -477,9 +477,9 @@ function generate_shim(pkgname, app::AppInfo, env, julia) end -function bash_shim(julia_escaped::String, module_spec_escaped::String, env) +function shell_shim(julia_escaped::String, module_spec_escaped::String, env) return """ - #!/usr/bin/env bash + #!/bin/sh $SHIM_HEADER From 12bc31e91a94f72913602bc38e9b3f85f2edc948 Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Thu, 3 Jul 2025 12:13:09 +0200 Subject: [PATCH 16/19] Enhance fuzzy matching algorithm with multi-factor scoring (#4287) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Rewrite the fuzzy sorting algorithm to provide more intuitive package name suggestions: Prefix matching prioritized for exact starts (e.g., "Plo" → "Plots") Subsequence matching with position weighting for scattered matches Weighted edit distance with common typo handling (a↔e, c↔k, etc.) Case preservation scoring and length penalties Optional popularity weighting for future use Co-authored-by: KristofferC Co-authored-by: Claude (cherry picked from commit 3869055abc37a360ccf52b268975005dec7f237f) --- CHANGELOG.md | 1 + src/Types.jl | 1 - src/fuzzysorting.jl | 356 ++++++++++++++++++++++++++++++-------------- test/new.jl | 23 ++- 4 files changed, 266 insertions(+), 115 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 7f39463873..5d2a754456 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ Pkg v1.12 Release Notes The functions `Pkg.status`, `Pkg.why`, `Pkg.instantiate`, `Pkg.precompile` (and their REPL variants) have been updated to take a `workspace` option. Read more about this feature in the manual about the TOML-files. - `status` now shows when different versions/sources of dependencies are loaded than that which is expected by the manifest ([#4109]) +- Enhanced fuzzy matching algorithm for package name suggestions. Pkg v1.11 Release Notes ======================= diff --git a/src/Types.jl b/src/Types.jl index eb6ca21eff..46260a3997 100644 --- a/src/Types.jl +++ b/src/Types.jl @@ -1236,7 +1236,6 @@ function write_env(env::EnvCache; update_undo=true, end end end - if (env.project != env.original_project) && (!skip_writing_project) write_project(env) end diff --git a/src/fuzzysorting.jl b/src/fuzzysorting.jl index 0d8d842b7f..ba2d8f82fd 100644 --- a/src/fuzzysorting.jl +++ b/src/fuzzysorting.jl @@ -2,129 +2,283 @@ module FuzzySorting _displaysize(io::IO) = displaysize(io)::Tuple{Int,Int} -# This code is duplicated from REPL.jl -# Considering breaking this into an independent package +# Character confusion weights for fuzzy matching +const CHARACTER_CONFUSIONS = Dict( + ('a', 'e') => 0.5, ('e', 'a') => 0.5, + ('i', 'y') => 0.5, ('y', 'i') => 0.5, + ('u', 'o') => 0.5, ('o', 'u') => 0.5, + ('c', 'k') => 0.3, ('k', 'c') => 0.3, + ('s', 'z') => 0.3, ('z', 's') => 0.3, + # Keyboard proximity (QWERTY layout) + ('q', 'w') => 0.4, ('w', 'q') => 0.4, + ('w', 'e') => 0.4, ('e', 'w') => 0.4, + ('e', 'r') => 0.4, ('r', 'e') => 0.4, + ('r', 't') => 0.4, ('t', 'r') => 0.4, + ('t', 'y') => 0.4, ('y', 't') => 0.4, + ('y', 'u') => 0.4, ('u', 'y') => 0.4, + ('u', 'i') => 0.4, ('i', 'u') => 0.4, + ('i', 'o') => 0.4, ('o', 'i') => 0.4, + ('o', 'p') => 0.4, ('p', 'o') => 0.4, + ('a', 's') => 0.4, ('s', 'a') => 0.4, + ('s', 'd') => 0.4, ('d', 's') => 0.4, + ('d', 'f') => 0.4, ('f', 'd') => 0.4, + ('f', 'g') => 0.4, ('g', 'f') => 0.4, + ('g', 'h') => 0.4, ('h', 'g') => 0.4, + ('h', 'j') => 0.4, ('j', 'h') => 0.4, + ('j', 'k') => 0.4, ('k', 'j') => 0.4, + ('k', 'l') => 0.4, ('l', 'k') => 0.4, + ('z', 'x') => 0.4, ('x', 'z') => 0.4, + ('x', 'c') => 0.4, ('c', 'x') => 0.4, + ('c', 'v') => 0.4, ('v', 'c') => 0.4, + ('v', 'b') => 0.4, ('b', 'v') => 0.4, + ('b', 'n') => 0.4, ('n', 'b') => 0.4, + ('n', 'm') => 0.4, ('m', 'n') => 0.4, +) -# Search & Rescue -# Utilities for correcting user mistakes and (eventually) -# doing full documentation searches from the repl. +# Enhanced fuzzy scoring with multiple factors +function fuzzyscore(needle::AbstractString, haystack::AbstractString) + needle_lower, haystack_lower = lowercase(needle), lowercase(haystack) -# Fuzzy Search Algorithm + # Factor 1: Prefix matching bonus (highest priority) + prefix_score = prefix_match_score(needle_lower, haystack_lower) -function matchinds(needle, haystack; acronym::Bool = false) - chars = collect(needle) - is = Int[] - lastc = '\0' - for (i, char) in enumerate(haystack) - while !isempty(chars) && isspace(first(chars)) - popfirst!(chars) # skip spaces - end - isempty(chars) && break - if lowercase(char) == lowercase(chars[1]) && - (!acronym || !isletter(lastc)) - push!(is, i) - popfirst!(chars) + # Factor 2: Subsequence matching + subseq_score = subsequence_score(needle_lower, haystack_lower) + + # Factor 3: Character-level similarity (improved edit distance) + char_score = character_similarity_score(needle_lower, haystack_lower) + + # Factor 4: Case preservation bonus + case_score = case_preservation_score(needle, haystack) + + # Factor 5: Length penalty for very long matches + length_penalty = length_penalty_score(needle, haystack) + + # Weighted combination + base_score = 0.4 * prefix_score + 0.3 * subseq_score + 0.2 * char_score + 0.1 * case_score + final_score = base_score * length_penalty + + return final_score +end + +# Prefix matching: exact prefix gets maximum score +function prefix_match_score(needle::AbstractString, haystack::AbstractString) + if startswith(haystack, needle) + return 1.0 + elseif startswith(needle, haystack) + return 0.9 # Partial prefix match + else + # Check for prefix after common separators + for sep in ['_', '-', '.'] + parts = split(haystack, sep) + for part in parts + if startswith(part, needle) + return 0.7 # Component prefix match + end + end end - lastc = char + return 0.0 end - return is end -longer(x, y) = length(x) ≥ length(y) ? (x, true) : (y, false) +# Subsequence matching with position weighting +function subsequence_score(needle::AbstractString, haystack::AbstractString) + if isempty(needle) + return 1.0 + end -bestmatch(needle, haystack) = - longer(matchinds(needle, haystack, acronym = true), - matchinds(needle, haystack)) + needle_chars = collect(needle) + haystack_chars = collect(haystack) -# Optimal string distance: Counts the minimum number of insertions, deletions, -# transpositions or substitutions to go from one string to the other. -function string_distance(a::AbstractString, lena::Integer, b::AbstractString, lenb::Integer) - if lena > lenb - a, b = b, a - lena, lenb = lenb, lena - end - start = 0 - for (i, j) in zip(a, b) - if a == b - start += 1 - else - break + matched_positions = Int[] + haystack_idx = 1 + + for needle_char in needle_chars + found = false + for i in haystack_idx:length(haystack_chars) + if haystack_chars[i] == needle_char + push!(matched_positions, i) + haystack_idx = i + 1 + found = true + break + end + end + if !found + return 0.0 end end - start == lena && return lenb - start - vzero = collect(1:(lenb - start)) - vone = similar(vzero) - prev_a, prev_b = first(a), first(b) - current = 0 - for (i, ai) in enumerate(a) - i > start || (prev_a = ai; continue) - left = i - start - 1 - current = i - start - transition_next = 0 - for (j, bj) in enumerate(b) - j > start || (prev_b = bj; continue) - # No need to look beyond window of lower right diagonal - above = current - this_transition = transition_next - transition_next = vone[j - start] - vone[j - start] = current = left - left = vzero[j - start] - if ai != bj - # Minimum between substitution, deletion and insertion - current = min(current + 1, above + 1, left + 1) - if i > start + 1 && j > start + 1 && ai == prev_b && prev_a == bj - current = min(current, (this_transition += 1)) - end - end - vzero[j - start] = current - prev_b = bj + + # Calculate score based on how clustered the matches are + if length(matched_positions) <= 1 + return 1.0 + end + + # Penalize large gaps between matches + gaps = diff(matched_positions) + avg_gap = sum(gaps) / length(gaps) + gap_penalty = 1.0 / (1.0 + avg_gap / 3.0) + + # Bonus for matches at word boundaries + boundary_bonus = 0.0 + for pos in matched_positions + if pos == 1 || haystack_chars[pos-1] in ['_', '-', '.'] + boundary_bonus += 0.1 end - prev_a = ai end - current -end -function fuzzyscore(needle::AbstractString, haystack::AbstractString) - lena, lenb = length(needle), length(haystack) - 1 - (string_distance(needle, lena, haystack, lenb) / max(lena, lenb)) + coverage = length(needle) / length(haystack) + return min(1.0, gap_penalty + boundary_bonus) * coverage end -function fuzzysort(search::String, candidates::Vector{String}) - scores = map(cand -> (FuzzySorting.fuzzyscore(search, cand), -Float64(FuzzySorting.levenshtein(search, cand))), candidates) - candidates[sortperm(scores)] |> reverse, any(s -> s[1] >= print_score_threshold, scores) +# Improved character-level similarity +function character_similarity_score(needle::AbstractString, haystack::AbstractString) + if isempty(needle) || isempty(haystack) + return 0.0 + end + + # Use Damerau-Levenshtein distance with character confusion weights + distance = weighted_edit_distance(needle, haystack) + max_len = max(length(needle), length(haystack)) + + return max(0.0, 1.0 - distance / max_len) end -# Levenshtein Distance +# Weighted edit distance accounting for common typos +function weighted_edit_distance(s1::AbstractString, s2::AbstractString) -function levenshtein(s1, s2) a, b = collect(s1), collect(s2) - m = length(a) - n = length(b) - d = Matrix{Int}(undef, m+1, n+1) + m, n = length(a), length(b) + # Initialize distance matrix + d = Matrix{Float64}(undef, m+1, n+1) d[1:m+1, 1] = 0:m d[1, 1:n+1] = 0:n for i = 1:m, j = 1:n - d[i+1,j+1] = min(d[i , j+1] + 1, - d[i+1, j ] + 1, - d[i , j ] + (a[i] != b[j])) + if a[i] == b[j] + d[i+1, j+1] = d[i, j] # No cost for exact match + else + # Standard operations + insert_cost = d[i, j+1] + 1.0 + delete_cost = d[i+1, j] + 1.0 + + # Check for repeated character deletion (common typo) + if i > 1 && a[i] == a[i-1] && a[i-1] == b[j] + delete_cost = d[i, j+1] + 0.3 # Low cost for deleting repeated char + end + + # Check for repeated character insertion (common typo) + if j > 1 && b[j] == b[j-1] && a[i] == b[j-1] + insert_cost = d[i, j+1] + 0.3 # Low cost for inserting repeated char + end + + # Substitution with confusion weighting + confusion_key = (a[i], b[j]) + subst_cost = d[i, j] + get(CHARACTER_CONFUSIONS, confusion_key, 1.0) + + d[i+1, j+1] = min(insert_cost, delete_cost, subst_cost) + + # Transposition + if i > 1 && j > 1 && a[i] == b[j-1] && a[i-1] == b[j] + d[i+1, j+1] = min(d[i+1, j+1], d[i-1, j-1] + 1.0) + end + end end return d[m+1, n+1] end -function levsort(search::String, candidates::Vector{String}) - scores = map(cand -> (Float64(levenshtein(search, cand)), -fuzzyscore(search, cand)), candidates) - candidates = candidates[sortperm(scores)] - i = 0 - for outer i = 1:length(candidates) - levenshtein(search, candidates[i]) > 3 && break +# Case preservation bonus +function case_preservation_score(needle::AbstractString, haystack::AbstractString) + if isempty(needle) || isempty(haystack) + return 0.0 + end + + matches = 0 + min_len = min(length(needle), length(haystack)) + + for i in 1:min_len + if needle[i] == haystack[i] + matches += 1 + end + end + + return matches / min_len +end + +# Length penalty for very long matches +function length_penalty_score(needle::AbstractString, haystack::AbstractString) + needle_len = length(needle) + haystack_len = length(haystack) + + if needle_len == 0 + return 0.0 + end + + # Strong preference for similar lengths + length_ratio = haystack_len / needle_len + length_diff = abs(haystack_len - needle_len) + + # Bonus for very close lengths (within 1-2 characters) + if length_diff <= 1 + return 1.1 # Small bonus for near-exact length + elseif length_diff <= 2 + return 1.05 + elseif length_ratio <= 1.5 + return 1.0 + elseif length_ratio <= 2.0 + return 0.8 + elseif length_ratio <= 3.0 + return 0.6 + else + return 0.4 # Heavy penalty for very long matches end - return candidates[1:i] end -# Result printing +# Main sorting function with optional popularity weighting +function fuzzysort(search::String, candidates::Vector{String}; popularity_weights::Dict{String,Float64} = Dict{String,Float64}()) + scores = map(candidates) do cand + base_score = fuzzyscore(search, cand) + weight = get(popularity_weights, cand, 1.0) + score = base_score * weight + return (score, cand) + end + + # Sort by score descending, then by candidate name for ties + sorted_scores = sort(scores, by = x -> (-x[1], x[2])) + + # Extract candidates and check if any meet threshold + result_candidates = [x[2] for x in sorted_scores] + has_good_matches = any(x -> x[1] >= print_score_threshold, sorted_scores) + + return result_candidates, has_good_matches +end + +# Keep existing interface functions for compatibility +function matchinds(needle, haystack; acronym::Bool = false) + chars = collect(needle) + is = Int[] + lastc = '\0' + for (i, char) in enumerate(haystack) + while !isempty(chars) && isspace(first(chars)) + popfirst!(chars) # skip spaces + end + isempty(chars) && break + if lowercase(char) == lowercase(chars[1]) && + (!acronym || !isletter(lastc)) + push!(is, i) + popfirst!(chars) + end + lastc = char + end + return is +end + +longer(x, y) = length(x) ≥ length(y) ? (x, true) : (y, false) + +bestmatch(needle, haystack) = + longer(matchinds(needle, haystack, acronym = true), + matchinds(needle, haystack)) function printmatch(io::IO, word, match) is, _ = bestmatch(word, match) @@ -137,7 +291,7 @@ function printmatch(io::IO, word, match) end end -const print_score_threshold = 0.5 +const print_score_threshold = 0.25 function printmatches(io::IO, word, matches; cols::Int = _displaysize(io)[2]) total = 0 @@ -152,25 +306,5 @@ end printmatches(args...; cols::Int = _displaysize(stdout)[2]) = printmatches(stdout, args..., cols = cols) -function print_joined_cols(io::IO, ss::Vector{String}, delim = "", last = delim; cols::Int = _displaysize(io)[2]) - i = 0 - total = 0 - for outer i = 1:length(ss) - total += length(ss[i]) - total + max(i-2,0)*length(delim) + (i>1 ? 1 : 0)*length(last) > cols && (i-=1; break) - end - join(io, ss[1:i], delim, last) -end - -print_joined_cols(args...; cols::Int = _displaysize(stdout)[2]) = print_joined_cols(stdout, args...; cols=cols) - -function print_correction(io::IO, word::String, mod::Module) - cors = map(quote_spaces, levsort(word, accessible(mod))) - pre = "Perhaps you meant " - print(io, pre) - print_joined_cols(io, cors, ", ", " or "; cols = _displaysize(io)[2] - length(pre)) - println(io) - return -end end diff --git a/test/new.jl b/test/new.jl index 108fd7039f..64717d1039 100644 --- a/test/new.jl +++ b/test/new.jl @@ -425,14 +425,31 @@ end ) Pkg.add(name="Example", rev="master", version="0.5.0") # Adding with a slight typo gives suggestions try - Pkg.add("Examplle") + io = IOBuffer() + Pkg.add("Examplle"; io) @test false # to fail if add doesn't error catch err @test err isa PkgError @test occursin("The following package names could not be resolved:", err.msg) @test occursin("Examplle (not found in project, manifest or registry)", err.msg) - @test occursin("Suggestions:", err.msg) - # @test occursin("Example", err.msg) # can't test this as each char in "Example" is individually colorized + @test occursin("Suggestions: Example", err.msg) + end + # Adding with lowercase suggests uppercase + try + io = IOBuffer() + Pkg.add("http"; io) + @test false # to fail if add doesn't error + catch err + @test err isa PkgError + @test occursin("Suggestions: HTTP", err.msg) + end + try + io = IOBuffer() + Pkg.add("Flix"; io) + @test false # to fail if add doesn't error + catch err + @test err isa PkgError + @test occursin("Suggestions: Flux", err.msg) end @test_throws PkgError( "name, UUID, URL, or filesystem path specification required when calling `add`" From 64015f7ad60092b3e13ad2b9694c37f5c3440b27 Mon Sep 17 00:00:00 2001 From: Keno Fischer Date: Mon, 30 Jun 2025 05:25:39 -0400 Subject: [PATCH 17/19] Registry: Properly pass down `depot` (#4268) (cherry picked from commit e02bcabd7443f6efec8d1add070cd8d85ef1347a) --- src/Pkg.jl | 12 ++++++++---- src/Registry/Registry.jl | 25 +++++++++++++++---------- test/registry.jl | 2 +- 3 files changed, 24 insertions(+), 15 deletions(-) diff --git a/src/Pkg.jl b/src/Pkg.jl index f3d58d2b4a..fba3b95092 100644 --- a/src/Pkg.jl +++ b/src/Pkg.jl @@ -21,10 +21,14 @@ public activate, add, build, compat, develop, free, gc, generate, instantiate, pin, precompile, redo, rm, resolve, status, test, undo, update, why depots() = Base.DEPOT_PATH -function depots1() - d = depots() - isempty(d) && Pkg.Types.pkgerror("no depots found in DEPOT_PATH") - return d[1] +function depots1(depot_list::Union{String, Vector{String}}=depots()) + # Get the first depot from a list, with proper error handling + if depot_list isa String + return depot_list + else + isempty(depot_list) && Pkg.Types.pkgerror("no depots provided") + return depot_list[1] + end end function pkg_server() diff --git a/src/Registry/Registry.jl b/src/Registry/Registry.jl index d5e938baa1..3477fc93fd 100644 --- a/src/Registry/Registry.jl +++ b/src/Registry/Registry.jl @@ -1,7 +1,7 @@ module Registry import ..Pkg -using ..Pkg: depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_server, +using ..Pkg: depots, depots1, printpkgstyle, stderr_f, isdir_nothrow, pathrepr, pkg_server, GitTools using ..Pkg.PlatformEngines: download_verify_unpack, download, download_verify, exe7z, verify_archive_tree_hash using UUIDs, LibGit2, TOML, Dates @@ -48,11 +48,11 @@ function add(; name=nothing, uuid=nothing, url=nothing, path=nothing, linked=not add([RegistrySpec(; name, uuid, url, path, linked)]; kwargs...) end end -function add(regs::Vector{RegistrySpec}; io::IO=stderr_f(), depot=depots1()) +function add(regs::Vector{RegistrySpec}; io::IO=stderr_f(), depots::Union{String, Vector{String}}=depots()) if isempty(regs) - download_default_registries(io, only_if_empty = false; depot) + download_default_registries(io, only_if_empty = false; depots=depots) else - download_registries(io, regs, depot) + download_registries(io, regs, depots) end end @@ -103,12 +103,15 @@ end pkg_server_url_hash(url::String) = Base.SHA1(split(url, '/')[end]) -function download_default_registries(io::IO; only_if_empty::Bool = true, depot=depots1()) - installed_registries = reachable_registries() +function download_default_registries(io::IO; only_if_empty::Bool = true, depots::Union{String, Vector{String}}=depots()) + # Check the specified depots for installed registries + installed_registries = reachable_registries(; depots) # Only clone if there are no installed registries, unless called # with false keyword argument. if isempty(installed_registries) || !only_if_empty - printpkgstyle(io, :Installing, "known registries into $(pathrepr(depot))") + # Install to the first depot in the list + target_depot = depots1(depots) + printpkgstyle(io, :Installing, "known registries into $(pathrepr(target_depot))") registries = copy(DEFAULT_REGISTRIES) for uuid in keys(pkg_server_registry_urls()) if !(uuid in (reg.uuid for reg in registries)) @@ -116,7 +119,7 @@ function download_default_registries(io::IO; only_if_empty::Bool = true, depot=d end end filter!(reg -> !(reg.uuid in installed_registries), registries) - download_registries(io, registries, depot) + download_registries(io, registries, depots) return true end return false @@ -168,9 +171,11 @@ function check_registry_state(reg) return nothing end -function download_registries(io::IO, regs::Vector{RegistrySpec}, depot::String=depots1()) +function download_registries(io::IO, regs::Vector{RegistrySpec}, depots::Union{String, Vector{String}}=depots()) + # Use the first depot as the target + target_depot = depots1(depots) populate_known_registries_with_urls!(regs) - regdir = joinpath(depot, "registries") + regdir = joinpath(target_depot, "registries") isdir(regdir) || mkpath(regdir) # only allow one julia process to download and install registries at a time FileWatching.mkpidlock(joinpath(regdir, ".pid"), stale_age = 10) do diff --git a/test/registry.jl b/test/registry.jl index 20d70ea038..96c4d1144a 100644 --- a/test/registry.jl +++ b/test/registry.jl @@ -257,7 +257,7 @@ end @test isempty(Registry.reachable_registries(; depots=[depot_off_path])) # After this, we have depots only in the depot that's off the path - Registry.add("General"; depot=depot_off_path) + Registry.add("General"; depots=depot_off_path) @test isempty(Registry.reachable_registries()) @test length(Registry.reachable_registries(; depots=[depot_off_path])) == 1 From f4e4b5044449161366218a63f0975bcd6cd75847 Mon Sep 17 00:00:00 2001 From: Ian Butterworth Date: Tue, 3 Jun 2025 11:52:12 -0400 Subject: [PATCH 18/19] Isolate threads test from parent state (#4243) Co-authored-by: gbaraldi (cherry picked from commit 5577f68d612139693282c037d070f515bf160d1b) --- test/new.jl | 64 +++++++++++-------- .../TestThreads/test/runtests.jl | 7 +- test/utils.jl | 2 +- 3 files changed, 45 insertions(+), 28 deletions(-) diff --git a/test/new.jl b/test/new.jl index 64717d1039..a87d510b37 100644 --- a/test/new.jl +++ b/test/new.jl @@ -2042,57 +2042,69 @@ end mktempdir() do dir path = copy_test_package(dir, "TestThreads") cd(path) do - with_current_env() do - default_nthreads_default = Threads.nthreads(:default) - default_nthreads_interactive = Threads.nthreads(:interactive) - other_nthreads_default = default_nthreads_default == 1 ? 2 : 1 - other_nthreads_interactive = default_nthreads_interactive == 0 ? 1 : 0 - @testset "default" begin + # Do this all in a subprocess to protect against the parent having non-default threadpool sizes. + script = """ + using Pkg, Test + @testset "JULIA_NUM_THREADS=1" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$default_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", + "EXPECTED_NUM_THREADS_DEFAULT" => "1", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "0", # https://github.com/JuliaLang/julia/pull/57454 + "JULIA_NUM_THREADS" => "1", ) do Pkg.test("TestThreads") end end - @testset "JULIA_NUM_THREADS=other_nthreads_default" begin + @testset "JULIA_NUM_THREADS=2" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", - "JULIA_NUM_THREADS" => "$other_nthreads_default", + "EXPECTED_NUM_THREADS_DEFAULT" => "2", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "1", + "JULIA_NUM_THREADS" => "2", ) do Pkg.test("TestThreads") end end - @testset "JULIA_NUM_THREADS=other_nthreads_default,other_nthreads_interactive" begin + @testset "JULIA_NUM_THREADS=2,0" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$other_nthreads_interactive", - "JULIA_NUM_THREADS" => "$other_nthreads_default,$other_nthreads_interactive", + "EXPECTED_NUM_THREADS_DEFAULT" => "2", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "0", + "JULIA_NUM_THREADS" => "2,0", ) do Pkg.test("TestThreads") end end - @testset "--threads=other_nthreads_default" begin + + @testset "--threads=1" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$default_nthreads_interactive", + "EXPECTED_NUM_THREADS_DEFAULT" => "1", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "0", # https://github.com/JuliaLang/julia/pull/57454 + "JULIA_NUM_THREADS" => nothing, ) do - Pkg.test("TestThreads"; julia_args=`--threads=$other_nthreads_default`) + Pkg.test("TestThreads"; julia_args=`--threads=1`) end end - @testset "--threads=other_nthreads_default,other_nthreads_interactive" begin + @testset "--threads=2" begin withenv( - "EXPECTED_NUM_THREADS_DEFAULT" => "$other_nthreads_default", - "EXPECTED_NUM_THREADS_INTERACTIVE" => "$other_nthreads_interactive", + "EXPECTED_NUM_THREADS_DEFAULT" => "2", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "1", + "JULIA_NUM_THREADS" => nothing, ) do - Pkg.test("TestThreads"; julia_args=`--threads=$other_nthreads_default,$other_nthreads_interactive`) + Pkg.test("TestThreads"; julia_args=`--threads=2`) end end - end + @testset "--threads=2,0" begin + withenv( + "EXPECTED_NUM_THREADS_DEFAULT" => "2", + "EXPECTED_NUM_THREADS_INTERACTIVE" => "0", + "JULIA_NUM_THREADS" => nothing, + ) do + Pkg.test("TestThreads"; julia_args=`--threads=2,0`) + end + end + """ + @test Utils.show_output_if_command_errors(`$(Base.julia_cmd()) --project=$(path) --startup-file=no -e "$script"`) end end - end + end end # diff --git a/test/test_packages/TestThreads/test/runtests.jl b/test/test_packages/TestThreads/test/runtests.jl index 43b8df8628..8d060b77f5 100644 --- a/test/test_packages/TestThreads/test/runtests.jl +++ b/test/test_packages/TestThreads/test/runtests.jl @@ -4,4 +4,9 @@ EXPECTED_NUM_THREADS_DEFAULT = parse(Int, ENV["EXPECTED_NUM_THREADS_DEFAULT"]) EXPECTED_NUM_THREADS_INTERACTIVE = parse(Int, ENV["EXPECTED_NUM_THREADS_INTERACTIVE"]) @assert Threads.nthreads() == EXPECTED_NUM_THREADS_DEFAULT @assert Threads.nthreads(:default) == EXPECTED_NUM_THREADS_DEFAULT -@assert Threads.nthreads(:interactive) == EXPECTED_NUM_THREADS_INTERACTIVE +if Threads.nthreads() == 1 + @info "Convert me back to an assert once https://github.com/JuliaLang/julia/pull/57454 has landed" Threads.nthreads(:interactive) EXPECTED_NUM_THREADS_INTERACTIVE +else + @assert Threads.nthreads(:interactive) == EXPECTED_NUM_THREADS_INTERACTIVE +end + diff --git a/test/utils.jl b/test/utils.jl index 7bbc606ade..473c8f5dc7 100644 --- a/test/utils.jl +++ b/test/utils.jl @@ -335,7 +335,7 @@ function show_output_if_command_errors(cmd::Cmd) println(read(out, String)) Base.pipeline_error(proc) end - return nothing + return true end function recursive_rm_cov_files(rootdir::String) From b3375c5f9de00aaf3515d6e636c277eb6e1bf4ba Mon Sep 17 00:00:00 2001 From: Kristoffer Carlsson Date: Tue, 1 Jul 2025 20:56:17 +0200 Subject: [PATCH 19/19] collect paths for input packages that are in a workspace (#4229) Co-authored-by: KristofferC (cherry picked from commit 2097cdb86a5063f065067fc5e0baba17b84d95e8) --- src/API.jl | 53 +++++++++---------- .../WorkspacePathResolution/Project.toml | 5 ++ .../SubProjectA/Project.toml | 9 ++++ .../SubProjectA/src/SubProjectA.jl | 7 +++ .../SubProjectB/Project.toml | 3 ++ .../SubProjectB/src/SubProjectB.jl | 5 ++ test/workspaces.jl | 15 ++++++ 7 files changed, 68 insertions(+), 29 deletions(-) create mode 100644 test/test_packages/WorkspacePathResolution/Project.toml create mode 100644 test/test_packages/WorkspacePathResolution/SubProjectA/Project.toml create mode 100644 test/test_packages/WorkspacePathResolution/SubProjectA/src/SubProjectA.jl create mode 100644 test/test_packages/WorkspacePathResolution/SubProjectB/Project.toml create mode 100644 test/test_packages/WorkspacePathResolution/SubProjectB/src/SubProjectB.jl diff --git a/src/API.jl b/src/API.jl index 78eaaa419d..9f39b225f7 100644 --- a/src/API.jl +++ b/src/API.jl @@ -187,35 +187,30 @@ for f in (:develop, :add, :rm, :up, :pin, :free, :test, :build, :status, :why, : end end -function update_source_if_set(project, pkg) +function update_source_if_set(env, pkg) + project = env.project source = get(project.sources, pkg.name, nothing) - source === nothing && return - # This should probably not modify the dicts directly... - if pkg.repo.source !== nothing - source["url"] = pkg.repo.source - end - if pkg.repo.rev !== nothing - source["rev"] = pkg.repo.rev - end - if pkg.path !== nothing - source["path"] = pkg.path - end - if pkg.subdir !== nothing - source["subdir"] = pkg.subdir - end - path, repo = get_path_repo(project, pkg.name) - if path !== nothing - pkg.path = path - end - if repo.source !== nothing - pkg.repo.source = repo.source - end - if repo.rev !== nothing - pkg.repo.rev = repo.rev + if source !== nothing + # This should probably not modify the dicts directly... + if pkg.repo.source !== nothing + source["url"] = pkg.repo.source + end + if pkg.repo.rev !== nothing + source["rev"] = pkg.repo.rev + end + if pkg.path !== nothing + source["path"] = pkg.path + end end - if repo.subdir !== nothing - pkg.repo.subdir = repo.subdir + + # Packages in manifest should have their paths set to the path in the manifest + for (path, wproj) in env.workspace + if wproj.uuid == pkg.uuid + pkg.path = Types.relative_project_path(env.manifest_file, dirname(path)) + break + end end + return end function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true, @@ -257,7 +252,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}; shared::Bool=true, if length(findall(x -> x.uuid == pkg.uuid, pkgs)) > 1 pkgerror("it is invalid to specify multiple packages with the same UUID: $(err_rep(pkg))") end - update_source_if_set(ctx.env.project, pkg) + update_source_if_set(ctx.env, pkg) end Operations.develop(ctx, pkgs, new_git; preserve=preserve, platform=platform) @@ -311,7 +306,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}; preserve::PreserveLevel=Op if length(findall(x -> x.uuid == pkg.uuid, pkgs)) > 1 pkgerror("it is invalid to specify multiple packages with the same UUID: $(err_rep(pkg))") end - update_source_if_set(ctx.env.project, pkg) + update_source_if_set(ctx.env, pkg) end Operations.add(ctx, pkgs, new_git; allow_autoprecomp, preserve, platform, target) @@ -390,7 +385,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}; ensure_resolved(ctx, ctx.env.manifest, pkgs) end for pkg in pkgs - update_source_if_set(ctx.env.project, pkg) + update_source_if_set(ctx.env, pkg) end Operations.up(ctx, pkgs, level; skip_writing_project, preserve) return diff --git a/test/test_packages/WorkspacePathResolution/Project.toml b/test/test_packages/WorkspacePathResolution/Project.toml new file mode 100644 index 0000000000..793ff0e389 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/Project.toml @@ -0,0 +1,5 @@ +[workspace] +projects = [ + "SubProjectA", + "SubProjectB", +] \ No newline at end of file diff --git a/test/test_packages/WorkspacePathResolution/SubProjectA/Project.toml b/test/test_packages/WorkspacePathResolution/SubProjectA/Project.toml new file mode 100644 index 0000000000..e5aa2bbe50 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/SubProjectA/Project.toml @@ -0,0 +1,9 @@ +name = "SubProjectA" +uuid = "87654321-4321-4321-4321-210987654321" +version = "0.1.0" + +[deps] +SubProjectB = "12345678-1234-1234-1234-123456789012" + +[sources] +SubProjectB = {path = "SubProjectB"} diff --git a/test/test_packages/WorkspacePathResolution/SubProjectA/src/SubProjectA.jl b/test/test_packages/WorkspacePathResolution/SubProjectA/src/SubProjectA.jl new file mode 100644 index 0000000000..06b4efa7b4 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/SubProjectA/src/SubProjectA.jl @@ -0,0 +1,7 @@ +module SubProjectA + +using SubProjectB + +greet() = "Hello from SubProjectA! " * SubProjectB.greet() + +end \ No newline at end of file diff --git a/test/test_packages/WorkspacePathResolution/SubProjectB/Project.toml b/test/test_packages/WorkspacePathResolution/SubProjectB/Project.toml new file mode 100644 index 0000000000..f989c45989 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/SubProjectB/Project.toml @@ -0,0 +1,3 @@ +name = "SubProjectB" +uuid = "12345678-1234-1234-1234-123456789012" +version = "0.1.0" \ No newline at end of file diff --git a/test/test_packages/WorkspacePathResolution/SubProjectB/src/SubProjectB.jl b/test/test_packages/WorkspacePathResolution/SubProjectB/src/SubProjectB.jl new file mode 100644 index 0000000000..342bdf6c94 --- /dev/null +++ b/test/test_packages/WorkspacePathResolution/SubProjectB/src/SubProjectB.jl @@ -0,0 +1,5 @@ +module SubProjectB + +greet() = "Hello from SubProjectB!" + +end \ No newline at end of file diff --git a/test/workspaces.jl b/test/workspaces.jl index 33b75d2baa..ba848246f0 100644 --- a/test/workspaces.jl +++ b/test/workspaces.jl @@ -163,4 +163,19 @@ end end end +@testset "workspace path resolution issue #4222" begin + mktempdir() do dir + path = copy_test_package(dir, "WorkspacePathResolution") + cd(path) do + with_current_env() do + # First resolve SubProjectB (non-root project) without existing Manifest + Pkg.activate("SubProjectB") + @test !isfile("Manifest.toml") + # Should be able to find SubProjectA and succeed + Pkg.update() + end + end + end +end + end # module