Merge pull request #4265 from JuliaLang/backports-release-1.12

Author: IanButterworth

src/API.jl

@@ -389,7 +389,9 @@ function up(ctx::Context, pkgs::Vector{PackageSpec};
         manifest_resolve!(ctx.env.manifest, pkgs)
         ensure_resolved(ctx, ctx.env.manifest, pkgs)
     end
-
+    for pkg in pkgs
+        update_source_if_set(ctx.env.project, pkg)
+    end
     Operations.up(ctx, pkgs, level; skip_writing_project, preserve)
     return
 end

src/Apps/Apps.jl

@@ -78,7 +78,6 @@ function _resolve(manifest::Manifest, pkgname=nothing)
         end
 
         projectfile = joinpath(app_env_folder(), pkg.name, "Project.toml")
-        sourcepath = source_path(app_manifest_file(), pkg)
 
         # TODO: Add support for existing manifest
         # Create a manifest with the manifest entry
@@ -87,12 +86,8 @@ function _resolve(manifest::Manifest, pkgname=nothing)
             if isempty(ctx.env.project.deps)
                 ctx.env.project.deps[pkg.name] = uuid
             end
-            if isempty(ctx.env.manifest)
-                ctx.env.manifest.deps[uuid] = pkg
-                Pkg.resolve(ctx)
-            else
-                Pkg.instantiate(ctx)
-            end
+            ctx.env.manifest.deps[uuid] = pkg
+            Pkg.resolve(ctx)
         end
 
         # TODO: Julia path

src/Operations.jl

@@ -222,21 +222,42 @@ end
 # This has to be done after the packages have been downloaded
 # since we need access to the Project file to read the information
 # about extensions
-function fixups_from_projectfile!(env::EnvCache)
+function fixups_from_projectfile!(ctx::Context)
+    env = ctx.env
     for pkg in values(env.manifest)
-        # isfile_casesenstive within locate_project_file used to error on Windows if given a
-        # relative path so abspath it to be extra safe https://github.com/JuliaLang/julia/pull/55220
-        project_file = Base.locate_project_file(abspath(source_path(env.manifest_file, pkg)))
-        if project_file isa String && isfile(project_file)
-            p = Types.read_project(project_file)
-            pkg.weakdeps = p.weakdeps
-            pkg.exts = p.exts
-            pkg.entryfile = p.entryfile
-            for (name, _) in p.weakdeps
+        if ctx.julia_version !== VERSION && is_stdlib(pkg.uuid, ctx.julia_version)
+            # Special handling for non-current julia_version resolving given the source for historical stdlibs
+            # isn't available at this stage as Pkg thinks it should not be needed, so rely on STDLIBS_BY_VERSION
+            stdlibs = Types.get_last_stdlibs(ctx.julia_version)
+            p = stdlibs[pkg.uuid]
+            pkg.weakdeps = Dict{String, Base.UUID}(stdlibs[uuid].name => uuid for uuid in p.weakdeps)
+            # pkg.exts = p.exts # TODO: STDLIBS_BY_VERSION doesn't record this
+            # pkg.entryfile = p.entryfile # TODO: STDLIBS_BY_VERSION doesn't record this
+            for (name, _) in pkg.weakdeps
                 if !haskey(p.deps, name)
                     delete!(pkg.deps, name)
                 end
             end
+        else
+            # normal mode based on project files.
+            # isfile_casesenstive within locate_project_file used to error on Windows if given a
+            # relative path so abspath it to be extra safe https://github.com/JuliaLang/julia/pull/55220
+            sourcepath = source_path(env.manifest_file, pkg)
+            if sourcepath === nothing
+                pkgerror("could not find source path for package $(pkg.name) based on manifest $(env.manifest_file)")
+            end
+            project_file = Base.locate_project_file(abspath(sourcepath))
+            if project_file isa String && isfile(project_file)
+                p = Types.read_project(project_file)
+                pkg.weakdeps = p.weakdeps
+                pkg.exts = p.exts
+                pkg.entryfile = p.entryfile
+                for (name, _) in p.weakdeps
+                    if !haskey(p.deps, name)
+                        delete!(pkg.deps, name)
+                    end
+                end
+            end
         end
     end
     prune_manifest(env)
@@ -337,18 +358,22 @@ function collect_developed!(env::EnvCache, pkg::PackageSpec, developed::Vector{P
     source = project_rel_path(env, source_path(env.manifest_file, pkg))
     source_env = EnvCache(projectfile_path(source))
     pkgs = load_project_deps(source_env.project, source_env.project_file, source_env.manifest, source_env.manifest_file)
-    for pkg in filter(is_tracking_path, pkgs)
+    for pkg in pkgs
         if any(x -> x.uuid == pkg.uuid, developed)
             continue
         end
-        # normalize path
-        # TODO: If path is collected from project, it is relative to the project file
-        # otherwise relative to manifest file....
-        pkg.path = Types.relative_project_path(env.manifest_file,
-                   project_rel_path(source_env,
-                   source_path(source_env.manifest_file, pkg)))
-        push!(developed, pkg)
-        collect_developed!(env, pkg, developed)
+        if is_tracking_path(pkg)
+            # normalize path
+            # TODO: If path is collected from project, it is relative to the project file
+             # otherwise relative to manifest file....
+            pkg.path = Types.relative_project_path(env.manifest_file,
+                    project_rel_path(source_env,
+                    source_path(source_env.manifest_file, pkg)))
+            push!(developed, pkg)
+            collect_developed!(env, pkg, developed)
+        elseif is_tracking_repo(pkg)
+            push!(developed, pkg)
+        end
     end
 end
 
@@ -1537,6 +1562,12 @@ function is_all_registered(registries::Vector{Registry.RegistryInstance}, pkgs::
 end
 
 function check_registered(registries::Vector{Registry.RegistryInstance}, pkgs::Vector{PackageSpec})
+    if isempty(registries) && !isempty(pkgs)
+        registry_pkgs = filter(tracking_registered_version, pkgs)
+        if !isempty(registry_pkgs)
+            pkgerror("no registries have been installed. Cannot resolve the following packages:\n$(join(map(pkg -> "  " * err_rep(pkg), registry_pkgs), "\n"))")
+        end
+    end
     pkg = is_all_registered(registries, pkgs)
     if pkg isa PackageSpec
         pkgerror("expected package $(err_rep(pkg)) to be registered")
@@ -1658,7 +1689,7 @@ function add(ctx::Context, pkgs::Vector{PackageSpec}, new_git=Set{UUID}();
         man_pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version)
         update_manifest!(ctx.env, man_pkgs, deps_map, ctx.julia_version)
         new_apply = download_source(ctx)
-        fixups_from_projectfile!(ctx.env)
+        fixups_from_projectfile!(ctx)
 
         # After downloading resolutionary packages, search for (Julia)Artifacts.toml files
         # and ensure they are all downloaded and unpacked as well:
@@ -1705,7 +1736,7 @@ function develop(ctx::Context, pkgs::Vector{PackageSpec}, new_git::Set{UUID};
     pkgs, deps_map = _resolve(ctx.io, ctx.env, ctx.registries, pkgs, preserve, ctx.julia_version)
     update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version)
     new_apply = download_source(ctx)
-    fixups_from_projectfile!(ctx.env)
+    fixups_from_projectfile!(ctx)
     download_artifacts(ctx; platform=platform, julia_version=ctx.julia_version)
     write_env(ctx.env) # write env before building
     show_update(ctx.env, ctx.registries; io=ctx.io)
@@ -1720,7 +1751,9 @@ function up_load_versions!(ctx::Context, pkg::PackageSpec, entry::PackageEntry,
     entry.version !== nothing || return false # no version to set
     if entry.pinned || level == UPLEVEL_FIXED
         pkg.version = entry.version
-        pkg.tree_hash = entry.tree_hash
+        if pkg.path === nothing
+            pkg.tree_hash = entry.tree_hash
+        end
     elseif entry.repo.source !== nothing || source_repo.source !== nothing # repo packages have a version but are treated specially
         if source_repo.source !== nothing
             pkg.repo = source_repo
@@ -1846,7 +1879,7 @@ function up(ctx::Context, pkgs::Vector{PackageSpec}, level::UpgradeLevel;
     end
     update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version)
     new_apply = download_source(ctx)
-    fixups_from_projectfile!(ctx.env)
+    fixups_from_projectfile!(ctx)
     download_artifacts(ctx, julia_version=ctx.julia_version)
     write_env(ctx.env; skip_writing_project) # write env before building
     show_update(ctx.env, ctx.registries; io=ctx.io, hidden_upgrades_info = true)
@@ -1892,7 +1925,7 @@ function pin(ctx::Context, pkgs::Vector{PackageSpec})
 
     update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version)
     new = download_source(ctx)
-    fixups_from_projectfile!(ctx.env)
+    fixups_from_projectfile!(ctx)
     download_artifacts(ctx; julia_version=ctx.julia_version)
     write_env(ctx.env) # write env before building
     show_update(ctx.env, ctx.registries; io=ctx.io)
@@ -1940,7 +1973,7 @@ function free(ctx::Context, pkgs::Vector{PackageSpec}; err_if_free=true)
 
         update_manifest!(ctx.env, pkgs, deps_map, ctx.julia_version)
         new = download_source(ctx)
-        fixups_from_projectfile!(ctx.env)
+        fixups_from_projectfile!(ctx)
         download_artifacts(ctx)
         write_env(ctx.env) # write env before building
         show_update(ctx.env, ctx.registries; io=ctx.io)
@@ -2553,8 +2586,7 @@ end
 
 function is_package_downloaded(manifest_file::String, pkg::PackageSpec; platform=HostPlatform())
     sourcepath = source_path(manifest_file, pkg)
-    identifier = pkg.name !== nothing ? pkg.name : pkg.uuid
-    (sourcepath === nothing) && pkgerror("Could not locate the source code for the $(identifier) package. Are you trying to use a manifest generated by a different version of Julia?")
+    sourcepath === nothing && return false
     isdir(sourcepath) || return false
     check_artifacts_downloaded(sourcepath; platform) || return false
     return true

src/REPLMode/argument_parsers.jl

@@ -57,7 +57,7 @@ let url = raw"((git|ssh|http(s)?)|(git@[\w\-\.]+))(:(//)?)([\w\.@\:/\-~]+)(\.git
     name_uuid = raw"[^@\#\s:]+\s*=\s*[^@\#\s:]+",
 
     # Match a `#BRANCH` branch or tag specifier.
-    branch = raw"\#\s*[^@\#\s]*",
+    branch = raw"\#\s*[^@^:\s]+",
 
     # Match an `@VERSION` version specifier.
     version = raw"@\s*[^@\#\s]*",
@@ -94,19 +94,34 @@ function parse_package_args(args::Vector{PackageToken}; add_or_dev=false)::Vecto
     # check for and apply PackageSpec modifier (e.g. `#foo` or `@v1.0.2`)
     function apply_modifier!(pkg::PackageSpec, args::Vector{PackageToken})
         (isempty(args) || args[1] isa PackageIdentifier) && return
-        modifier = popfirst!(args)
-        if modifier isa Subdir
-            pkg.subdir = modifier.dir
-            (isempty(args) || args[1] isa PackageIdentifier) && return
+        parsed_subdir = false
+        parsed_version = false
+        parsed_rev = false
+        while !isempty(args)
             modifier = popfirst!(args)
-        end
-
-        if modifier isa VersionToken
-            pkg.version = modifier.version
-        elseif modifier isa Rev
-            pkg.rev = modifier.rev
-        else
-            pkgerror("Package name/uuid must precede subdir specifier `$args`.")
+            if modifier isa Subdir
+                if parsed_subdir
+                    pkgerror("Multiple subdir specifiers `$args` found.")
+                end
+                pkg.subdir = modifier.dir
+                (isempty(args) || args[1] isa PackageIdentifier) && return
+                modifier = popfirst!(args)
+                parsed_subdir = true
+            elseif modifier isa VersionToken
+                if parsed_version
+                    pkgerror("Multiple version specifiers `$args` found.")
+                end
+                pkg.version = modifier.version
+                parsed_version = true
+            elseif modifier isa Rev
+                if parsed_rev
+                    pkgerror("Multiple revision specifiers `$args` found.")
+                end
+                pkg.rev = modifier.rev
+                parsed_rev = true
+            else
+                pkgerror("Package name/uuid must precede subdir specifier `$args`.")
+            end
         end
     end
 

src/Types.jl

@@ -85,7 +85,8 @@ end
 
 Base.:(==)(r1::GitRepo, r2::GitRepo) =
     r1.source == r2.source && r1.rev == r2.rev && r1.subdir == r2.subdir
-
+Base.hash(r::GitRepo, h::UInt) =
+    foldr(hash, [r.source, r.rev, r.subdir], init=h)
 
 mutable struct PackageSpec
     name::Union{Nothing,String}
@@ -119,11 +120,15 @@ PackageSpec(name::AbstractString, uuid::UUID) = PackageSpec(;name=name, uuid=uui
 PackageSpec(name::AbstractString, version::VersionTypes) = PackageSpec(;name=name, version=version)::PackageSpec
 PackageSpec(n::AbstractString, u::UUID, v::VersionTypes) = PackageSpec(;name=n, uuid=u, version=v)::PackageSpec
 
+# XXX: These definitions are a bit fishy. It seems to be used in an `==` call in status printing
 function Base.:(==)(a::PackageSpec, b::PackageSpec)
     return a.name == b.name && a.uuid == b.uuid && a.version == b.version &&
     a.tree_hash == b.tree_hash && a.repo == b.repo && a.path == b.path &&
     a.pinned == b.pinned
 end
+function Base.hash(a::PackageSpec, h::UInt)
+    return foldr(hash, [a.name, a.uuid, a.version, a.tree_hash, a.repo, a.path, a.pinned], init=h)
+end
 
 function err_rep(pkg::PackageSpec)
     x = pkg.name !== nothing && pkg.uuid !== nothing ? x = "$(pkg.name) [$(string(pkg.uuid)[1:8])]" :

src/manifest.jl

@@ -288,6 +288,9 @@ function destructure(manifest::Manifest)::Dict
     end
 
     for (uuid, entry) in manifest
+        # https://github.com/JuliaLang/Pkg.jl/issues/4086
+        @assert !(entry.tree_hash !== nothing && entry.path !== nothing)
+
         new_entry = something(entry.other, Dict{String,Any}())
         new_entry["uuid"] = string(uuid)
         entry!(new_entry, "version", entry.version)

test/apps.jl

@@ -33,6 +33,14 @@ isolate(loaded_depot=true) do
             Pkg.Apps.rm("Rot13")
             @test Sys.which(exename) == nothing
         end
+
+        # https://github.com/JuliaLang/Pkg.jl/issues/4258
+        Pkg.Apps.add(path=path)
+        Pkg.Apps.develop(path=path)
+        mv(joinpath(path, "src", "Rot13_edited.jl"), joinpath(path, "src", "Rot13.jl"); force=true)
+        withenv("PATH" => string(joinpath(first(DEPOT_PATH), "bin"), sep, current_path)) do
+            @test read(`$exename test`, String) == "Updated!\n"
+        end
     end
 end