Merge #398

398: Use Pkg registries r=ericphanson a=SebastianM-C

This implements the suggestion in #395 and it should also adress #396 
By using the already stored registries the code is simplified and `clone_all_registries` is no longer needed. I have commented out the code and if that function is no longer needed, I will remove it.

The new implementation relies on the user depot, defined as the first entry of the `DEPOT_PATH` by the [Pkg manual](https://pkgdocs.julialang.org/v1/glossary/). Due to how registries are stored in the depot, I had to move the test registries in a `registries` folder. This doesn't seem to affect other tests.

I have tested this locally (in WSL) and I'm getting the following error
```julia
Initialized empty Git repository in /tmp/jl_Ir9E2F/.git/
error: refname refs/heads/master not found
fatal: Branch rename failed
master branch: Error During Test at /mnt/c/Users/sebastian/.julia/dev/CompatHelper/test/utilities/git.jl:349
  Got exception outside of a `@test`
  failed process: Process(`git branch -m main`, ProcessExited(128)) [128]
  
  Stacktrace:
    [1] pipeline_error
      @ ./process.jl:525 [inlined]
    [2] run(::Cmd; wait::Bool)
      @ Base ./process.jl:440
    [3] run
      @ ./process.jl:438 [inlined]
    [4] (::var"#94#98"{String})()
      @ Main /mnt/c/Users/sebastian/.julia/dev/CompatHelper/test/utilities/git.jl:354
    [5] cd(f::var"#94#98"{String}, dir::String)
      @ Base.Filesystem ./file.jl:106
    [6] #93
      @ /mnt/c/Users/sebastian/.julia/dev/CompatHelper/test/utilities/git.jl:352 [inlined]
    [7] mktempdir(fn::var"#93#97"{String}, parent::String; prefix::String)
      @ Base.Filesystem ./file.jl:729
    [8] mktempdir(fn::Function, parent::String) (repeats 2 times)
      @ Base.Filesystem ./file.jl:727
    [9] macro expansion
      @ /mnt/c/Users/sebastian/.julia/dev/CompatHelper/test/utilities/git.jl:351 [inlined]
   [10] macro expansion
      @ /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [11] macro expansion
      @ /mnt/c/Users/sebastian/.julia/dev/CompatHelper/test/utilities/git.jl:350 [inlined]
   [12] macro expansion
      @ /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [13] top-level scope
      @ /mnt/c/Users/sebastian/.julia/dev/CompatHelper/test/utilities/git.jl:333
   [14] include(fname::String)
      @ Base.MainInclude ./client.jl:444
   [15] macro expansion
      @ /mnt/c/Users/sebastian/.julia/dev/CompatHelper/test/runtests.jl:25 [inlined]
   [16] macro expansion
      @ /buildworker/worker/package_linux64/build/usr/share/julia/stdlib/v1.6/Test/src/Test.jl:1151 [inlined]
   [17] top-level scope
      @ /mnt/c/Users/sebastian/.julia/dev/CompatHelper/test/runtests.jl:24
   [18] include(fname::String)
      @ Base.MainInclude ./client.jl:444
   [19] top-level scope
      @ none:6
   [20] eval
      @ ./boot.jl:360 [inlined]
   [21] exec_options(opts::Base.JLOptions)
      @ Base ./client.jl:261
   [22] _start()
      @ Base ./client.jl:485
```
but I'm not sure if it's related to the changes in this PR.

Let me kow what you think about this approach. cc `@DilumAluthge` 

Co-authored-by: Sebastian Micluța-Câmpeanu <[email protected]>
Co-authored-by: Sebastian Micluța-Câmpeanu <[email protected]>

Author: bors[bot]

Project.toml

@@ -1,7 +1,7 @@
 name = "CompatHelper"
 uuid = "aa819f21-2bde-4658-8897-bab36330d9b7"
 authors = ["Dilum Aluthge", "Brown Center for Biomedical Informatics", "contributors"]
-version = "3.0.8"
+version = "3.1.0"
 
 [deps]
 Base64 = "2a0f44e3-6c83-55bd-87e4-b1978d98bd5f"

docs/src/options.md

@@ -26,6 +26,20 @@ my_registries = [Pkg.RegistrySpec(name = "General",
 CompatHelper.main(; registries=my_registries)
 ```
 
+Using the above option will clone the registries, but if you want to use already existing registries,
+you can use the `use_existing_registries` option. This will use all the registries present in the user depot,
+defined as the first entry of the `DEPOT_PATH` by the [Pkg manual](https://pkgdocs.julialang.org/v1/glossary/).
+```julia
+CompatHelper.main(; use_existing_registries=true)
+```
+
+If you want to use a different location, you can specify that using the `depot` keyword.
+Note that the directroy indicated by `depot` should contain a subdirectory named `registries` where the registries
+are stored in order to reproduce the structure of the Julia depot.
+
+!!! info
+    Using the above mentioned `use_existing_registries` can be used in conjunction with the [`add-julia-registry`](https://github.com/julia-actions/add-julia-registry) GitHub action to easily add custom private registries in GitHub Actions CI.
+
 ## Overriding the default branch
 
 By default, CompatHelper will open pull requests against your repository's default branch. If you would like to override this behavior, set the `master_branch` keyword argument. For example:
@@ -61,6 +75,5 @@ When a compat Pull Request is created, you might want the user that generated th
 This will use the `GITHUB_ACTOR` or `GITLAB_USER_LOGIN` environment variables to determine which user to mention.
 
 ```julia
-CompatHElper.main(; cc_user=true)
+CompatHelper.main(; cc_user=true)
 ```
-

src/dependencies.jl

@@ -76,30 +76,43 @@ function clone_all_registries(f::Function, registry_list::Vector{Pkg.RegistrySpe
     end
 end
 
+function get_latest_version!(deps, registries)
+    for registry in registries
+        registry_toml_path = joinpath(registry, "Registry.toml")
+        registry_toml = TOML.parsefile(registry_toml_path)
+        packages = registry_toml["packages"]
+
+        for dep in deps
+            uuid = string(dep.package.uuid)
+
+            if uuid in keys(packages)
+                versions_toml_path = joinpath(
+                    registry, packages[uuid]["path"], "Versions.toml"
+                )
+                versions =
+                    VersionNumber.(collect(keys(TOML.parsefile(versions_toml_path))))
+
+                max_version = maximum(versions)
+                dep.latest_version = _max(dep.latest_version, max_version)
+            end
+        end
+    end
+end
+
+function get_existing_registries!(
+    deps::Set{DepInfo}, depot::String
+)
+    registries = readdir(joinpath(depot, "registries"), join=true)
+    get_latest_version!(deps, registries)
+
+    return deps
+end
+
 function get_latest_version_from_registries!(
     deps::Set{DepInfo}, registry_list::Vector{Pkg.RegistrySpec}
 )
     @mock clone_all_registries(registry_list) do registry_temp_dirs
-        for registry in registry_temp_dirs
-            registry_toml_path = joinpath(registry, "Registry.toml")
-            registry_toml = TOML.parsefile(joinpath(registry_toml_path))
-            packages = registry_toml["packages"]
-
-            for dep in deps
-                uuid = string(dep.package.uuid)
-
-                if uuid in keys(packages)
-                    versions_toml_path = joinpath(
-                        registry, packages[uuid]["path"], "Versions.toml"
-                    )
-                    versions =
-                        VersionNumber.(collect(keys(TOML.parsefile(versions_toml_path))))
-
-                    max_version = maximum(versions)
-                    dep.latest_version = _max(dep.latest_version, max_version)
-                end
-            end
-        end
+        get_latest_version!(deps, registry_temp_dirs)
     end
 
     return deps

src/main.jl

@@ -10,6 +10,8 @@ const DEFAULT_REGISTRIES = Pkg.RegistrySpec[Pkg.RegistrySpec(;
         ci_cfg::CIService=auto_detect_ci_service(; env=env);
         entry_type::EntryType=KeepEntry(),
         registries::Vector{Pkg.RegistrySpec}=DEFAULT_REGISTRIES,
+        use_existing_registries::Bool=false,
+        depot::String=DEPOT_PATH[1],
         subdirs::AbstractVector{<:AbstractString}=[""],
         master_branch::Union{DefaultBranch,AbstractString}=DefaultBranch(),
         bump_compat_containing_equality_specifier=true,
@@ -29,6 +31,8 @@ Main entry point for the package.
 # Keywords
 - `entry_type::EntryType=KeepEntry()`: How to handle bumps for entry types
 - `registries::Vector{Pkg.RegistrySpec}=DEFAULT_REGISTRIES`: RegistrySpec of all registries to use
+- `use_existing_registries::Bool=false`: Specify whether to use the registries available at the `depot` location
+- `depot::String=DEPOT_PATH[1]`: The user depot path to use
 - `subdirs::AbstractVector{<:AbstractString}=[""]`: Subdirectories for nested packages
 - `master_branch::Union{DefaultBranch,AbstractString}=DefaultBranch()`: Name of the master branch
 - `bump_compat_containing_equality_specifier=true`: Bump compat entries with equality specifiers
@@ -43,6 +47,8 @@ function main(
     ci_cfg::CIService=auto_detect_ci_service(; env=env);
     entry_type::EntryType=KeepEntry(),
     registries::Vector{Pkg.RegistrySpec}=DEFAULT_REGISTRIES,
+    use_existing_registries::Bool=false,
+    depot::String=DEPOT_PATH[1],
     subdirs::AbstractVector{<:AbstractString}=[""],
     master_branch::Union{DefaultBranch,AbstractString}=DefaultBranch(),
     bump_compat_containing_equality_specifier=true,
@@ -65,7 +71,12 @@ function main(
             include_jll=include_jll,
             master_branch=master_branch,
         )
-        get_latest_version_from_registries!(deps, registries)
+
+        if use_existing_registries
+            get_existing_registries!(deps, depot)
+        else
+            get_latest_version_from_registries!(deps, registries)
+        end
 
         for dep in deps
             pr = @mock make_pr_for_new_version(

test/dependencies.jl

@@ -61,6 +61,7 @@ end
             CompatHelper.Package(packageC, UUID(2)); latest_version=VersionNumber("3")
         ),
     ])
+
     apply([clone_all_registries_patch, rm_patch]) do
         result = CompatHelper.get_latest_version_from_registries!(
             deps, Vector{Pkg.RegistrySpec}()
@@ -79,3 +80,40 @@ end
         end
     end
 end
+
+@testset "get_existing_registries!" begin
+    packageA = "PackageA"
+    packageB = "PackageB"
+    packageC = "PackageC"
+
+    deps = Set{CompatHelper.DepInfo}([
+        # No version specified
+        CompatHelper.DepInfo(CompatHelper.Package(packageA, UUID(0))),
+
+        # Version is less than what is in registry
+        CompatHelper.DepInfo(
+            CompatHelper.Package(packageB, UUID(1)); latest_version=VersionNumber(1)
+        ),
+
+        # Version is greater than what is in registry
+        CompatHelper.DepInfo(
+            CompatHelper.Package(packageC, UUID(2)); latest_version=VersionNumber("3")
+        ),
+    ])
+
+    result = CompatHelper.get_existing_registries!(
+        deps, joinpath(@__DIR__, "deps")
+    )
+
+    @test length(result) == 3
+
+    for res in result
+        if res.package.name == packageA
+            @test res.latest_version == VersionNumber("1")
+        elseif res.package.name == packageB
+            @test res.latest_version == VersionNumber("2")
+        elseif res.package.name == packageC
+            @test res.latest_version == VersionNumber("3")
+        end
+    end
+end