refactor again to make REGISTRY_PATH a user-defined, external, path

Author: ablaom

registry/Metadata.toml

@@ -116,7 +116,7 @@ end
 Assuming a package `environment` path is specified, do the following in a new Julia
 process:
 
-1.Activate `environment`.
+1. Activate `environment`.
 
 2. Evaluate the `setup` expression, if specified.
 

registry/Project.toml

@@ -3,38 +3,42 @@
 
 Module providing methods for managing the MLJ Model Registry. To modify the registry:
 
-- Create a local clone of the [MLJModelRegistry.jl
-  repository](https://github.com/JuliaAI/MLJModelRegistry.jl)
+!!! important
+
+    In any pull request to update the Model Registry you should note the final output of
+    `Pkg.status(outdated=true)` when you have MLJModels/registry activated.
+
+- Create a local clone of [MLJModels.jl](https://github.com/JuliaAI/MLJModels.jl), which
+  hosts the registry. After making changes, you will be making a MLJModels.jl pull
+  request.
 
 - Use Julia's package manager to add or remove items from the list of registered packages
-  in the environment "/registry/", located in the root directory of your clone. If adding
-  a new item, see the protocol below.
+  in the environment "MLJModels/registry/". If adding a new item, see the protocol below.
+
+- After adding MLJModelRegistry to some other Julia pkg environment you have activated
+(e.g., a fresh temporary one) run `using MLJModelRegistry` to make the management tools
+available.
 
-- In a fresh temporary environment, run `Pkg.develop("path_to_clone")` and `using
-  MLJModelRegistry`.
+- Point MLJModelRegistry to the location of the registry itself within your MLJModels.jl
+  clone, using `setpath(path_to_registry)`, as in `setpath("MyPkgs/MLJModels/registry")`.
 
 - To add or update the metadata associated with a package, run [`update(pkg)`](@ref).
 
 - Assuming this is successful, update the metadata for *all* packages in the registry
   by running [`update()`](@ref).
 
-- When satisfied, commit your changes to the clone and make a pull request to the master
-  MLJModelRegistry.jl repository.
+- When satisfied, commit your changes to the clone and make a pull request to the
+  MLJModelRegistry.jl repository that you cloned.
 
 !!! note
 
-    Removing a package from the "/registry/" enviroment does not remove its metadata from
-    the Model Registry (i.e., from "/registry/Metatdata.toml"). However if you call
-    `update()` to update all package metadata (or call [`MLJModelRegistry.gc()`](@ref))
-    the metadata for all orphaned packages is removed.
+    Removing a package from the registry environment does not remove its metadata (i.e.,
+    from "/registry/Metatdata.toml"). However if you call `update()` to update all package
+    metadata (or call [`MLJModelRegistry.ac()`](@ref)) the metadata for all orphaned
+    packages is removed.
 
 # Protocol for adding new packages to the registry environment
 
-!!! important
-
-    In any pull request to update the Model Registry you should note the final
-    output of `Pkg.status(outdated=true)`.
-
 1. In your local clone of MLJModelRegistry.jl, `activate` the environment at  "/registry/".
 
 2. `update` the environment
@@ -58,14 +62,18 @@ using InteractiveUtils
 using Suppressor
 using Distributed
 
-# Location of the MLJ model registry (a Julia pkg environment + metadata):
-const ROOT = joinpath(@__DIR__, "..")
-const REGISTRY = joinpath(ROOT, "registry")
-
 # for controlling logging:
 struct Loud end
 struct Quiet end
 
+const ROOT = joinpath(@__DIR__, "..")
+
+# initializes REGISTRY_PATH to "":
+include("init.jl")
+
+# setters and getters for REGISTRY_PATH:
+include("setpath.jl")
+
 # The MLJ Model Registry is a special case of a "generic model registry", as described in
 # this file, defining the `GenericRegistry` module (which has methods, no types):
 include("GenericRegistry.jl")
@@ -79,6 +87,6 @@ include("remote_methods.jl")
 # top-level methods for registry management:
 include("methods.jl")
 
-export update
+export update, setpath
 
 end # module

src/GenericRegistry.jl

@@ -0,0 +1,3 @@
+function __init__()
+    global REGISTRY_PATH=Ref("")
+end

src/MLJModelRegistry.jl

@@ -25,34 +25,60 @@ function clean!(dic, pkg)
     return dic
 end
 
+# develop MLJModelRegistry into the specifified `registry` project:
+function setup(registry)
+    ex = quote
+        # REMOVE THIS NEXT LINE AFTER TAGGING NEW MLJMODELINTERFACE
+        Pkg.develop(path="/Users/anthony/MLJ/MLJModelInterface/")
+        Pkg.develop(path=$ROOT) # MLJModelRegistry
+    end
+    future = GenericRegistry.run([], ex; environment=registry)
+    fetch(future)
+    GenericRegistry.close(future)
+end
+
+# remove MLJModelRegistry from the specifified `registry` project:
+function cleanup(registry)
+    ex = quote
+        Pkg.rm("MLJModelRegistry")
+    end
+    future = GenericRegistry.run([], ex; environment=registry)
+    fetch(future)
+    GenericRegistry.close(future)
+end
+
 """
-    metadata(pkg; manifest=true, check_traits=true)
+    metadata(pkg; registry="", check_traits=true)
 
 *Private method.*
 
 Extract the metadata for a package. Returns a `Future` object that must be `fetch`ed to
 get the metadata. See, [`MLJModelRegistry.update`](@ref), which calls this method, for
 more details.
 
+Assumes that MLJModelRegistry has been `develop`ed into `registry` if this is non-empty.
+
 """
-function metadata(pkg; environment=nothing, check_traits=true)
-    if !isnothing(environment)
-        pkg in GenericRegistry.dependencies(environment) ||
-            throw(err_missing_package(pkg, environment))
-    end
-    setup = quote
-        # REMOVE THIS NEXT LINE AFTER TAGGING NEW MLJMODELINTERFACE
-        Pkg.develop(path="/Users/anthony/MLJ/MLJModelInterface/")
-        Pkg.develop(path=$ROOT)
+function metadata(pkg; registry="", check_traits=true)
+    if !isempty(registry)
+        pkg in GenericRegistry.dependencies(registry) ||
+            throw(err_missing_package(pkg, registry))
+        setup=()
+    else
+        setup = quote
+            # REMOVE THIS NEXT LINE AFTER TAGGING NEW MLJMODELINTERFACE
+            Pkg.develop(path="/Users/anthony/MLJ/MLJModelInterface/")
+            Pkg.develop(path=$ROOT) # MLJModelRegistry
+        end
     end
     program = quote
         import MLJModelRegistry
-        dic = MLJModelRegistry.traits_given_constructor_name(
+        MLJModelRegistry.traits_given_constructor_name(
             $pkg,
             check_traits=$check_traits,
         )
     end
-    return GenericRegistry.run(setup, pkg, program; environment)
+    return GenericRegistry.run(setup, pkg, program; environment=registry)
 end
 
 
@@ -67,7 +93,7 @@ registry.
 This is performed automatically after `update()`, but not after `update(pkg)`.
 
 """
-gc() = GenericRegistry.gc(REGISTRY)
+gc() = GenericRegistry.gc(registry_path())
 
 
 """
@@ -78,28 +104,27 @@ strings, and record this in the MLJ model registry (write it to
 `/registry/Metadata.toml`).
 
 Assumes `pkg` is already a dependency in the Julia environment defined at `/registry/` and
-uses the version of `pkg` consistent with the current environment manifest. See
-documentation for details on the registration process.
+uses the version of `pkg` consistent with the current environment manifest, after
+MLJModelRegistry.jl has been `develop`ed into that environment (it is removed again after
+the update). See documentation for details on the registration process.
 
 ```julia-repl
 julia> update("MLJDecisionTreeInterface")
 ```
 
 # Return value
 
-A set of all names of all models (more precisely, constructors) for which metadata was
-recorded.
+The metadata dictionary, keyed on models (more precisely, constructors, thereof).
 
 # Advanced options
 
 !!! warning
 
     Advanced options are intented primarily for diagnostic purposes.
 
-- `manifest=true`: Set to `false` to ignore the registry environment manifest (at
-  `/registry/Manifest.toml`) and instead add only the specified packages to a new
-  temporary environment. Useful to temporarily force latest versions if these are being
-  blocked by other packages.
+- `manifest=true`: Set to `false` to ignore the registry environment manifest and instead
+  add only the specified packages to a new temporary environment. Useful to temporarily
+  force latest versions if these are being blocked by other packages.
 
 - `debug=false`: Calling `update` opens a temporary Julia process to extract the trait
   metadata (see [`MLJModelRegistry.GenericRegistry.run`](@ref)). By default, this process
@@ -109,23 +134,26 @@ recorded.
 
 """
 function update(pkg; debug=false, manifest=true, check_traits=true)
-    environment = manifest ? REGISTRY : nothing
+    registry = manifest ? registry_path() : ""
     @info INFO_BE_PATIENT1
-    update(pkg, debug ? Loud() : Quiet(), environment, check_traits)
+    update(pkg, debug ? Loud() : Quiet(), registry, check_traits)
 end
-update(pkg, ::Loud, environment, check_traits) = _update(pkg, environment, check_traits)
-update(pkg, ::Quiet, environment, check_traits) =
-    @suppress _update(pkg, environment, check_traits)
-function _update(pkg, environment, check_traits)
-    future = MLJModelRegistry.metadata(pkg; environment, check_traits)
+update(pkg, ::Loud, registry, check_traits) = _update(pkg, true, registry, check_traits)
+update(pkg, ::Quiet, registry, check_traits) =
+    @suppress _update(pkg, false, registry, check_traits)
+function _update(pkg, debug, registry, check_traits)
+    isempty(registry) || setup(registry)
+    future = MLJModelRegistry.metadata(pkg; registry, check_traits)
     metadata = try
         fetch(future)
     catch excptn
+        isempty(registry) || cleanup(registry)
         debug || GenericRegistry.close(future)
         rethrow(excptn)
     end
-    GenericRegistry.close(future)
-    GenericRegistry.put(pkg, metadata, REGISTRY)
+    isempty(registry) || cleanup(registry)
+    debug || GenericRegistry.close(future)
+    GenericRegistry.put(pkg, metadata, registry_path())
 end
 
 """
@@ -147,8 +175,10 @@ A set of all names of all packages for which metadata was recorded.
   updates in parallel. Metadata is extracted in parallel, but written to file
   sequentially.
 
-- `debug=false`, `manifest=true`: These are applied as indicated above for each package
-  added.
+- `debug=false`: Set to `true` to leave temporary processes open; see the `update(pkg;
+  ...)` document string above.
+
+- `manifest=true`: See the `update(pkg; ...)` document string above.
 
 """
 function update(
@@ -158,35 +188,38 @@ function update(
     manifest=true,
     check_traits=true,
     )
-    environment = manifest ? REGISTRY : nothing
-    allpkgs = GenericRegistry.dependencies(REGISTRY)
-    if !isnothing(environment)
-        issubset(skip, allpkgs) || throw(err_invalid_packages(skip, environment))
+    registry = manifest ? registry_path() : ""
+    allpkgs = GenericRegistry.dependencies(registry_path())
+    if !isempty(registry)
+        issubset(skip, allpkgs) || throw(err_invalid_packages(skip, registry))
+        @suppress setup(registry)
     end
     pkgs = setdiff(allpkgs, skip)
     N = length(pkgs)
     pos = 1
-    @info "Processing $nworkers packages at a time. "
+    @info "Processing up to $nworkers packages at a time. "
     @info INFO_BE_PATIENT10
     while N ≥ 1
         print("\rPackages remaining: $N ")
         n = min(nworkers, N)
         batch = pkgs[pos:pos + n - 1]
         @suppress begin
             futures =
-                [MLJModelRegistry.metadata(pkg; environment, check_traits) for pkg in batch]
+                [MLJModelRegistry.metadata(pkg; registry, check_traits) for pkg in batch]
             try
                 for (i, f) in enumerate(futures)
-                    GenericRegistry.put(batch[i], fetch(f), REGISTRY)
+                    GenericRegistry.put(batch[i], fetch(f), registry_path())
                 end
             catch excptn
+                isempty(registry) || cleanup(registry)
                 debug || GenericRegistry.close.(futures)
                 rethrow(excptn)
             end
-            GenericRegistry.close.(futures)
+            debug || GenericRegistry.close.(futures)
         end
         N -= n
     end
+    isempty(registry) || @suppress cleanup(registry)
     gc()
     println("\rPackages remaining: 0   ")
     return pkgs
@@ -200,4 +233,4 @@ Inspect the model trait metadata recorded in the Model Registry for those models
 see [`MLJModelRegistry.encode_dic`](@ref).
 
 """
-get(pkg) = GenericRegistry.get(pkg, REGISTRY)
+get(pkg) = GenericRegistry.get(pkg, registry_path())

src/init.jl

@@ -0,0 +1,16 @@
+# # LOGGING
+
+const ERR_REGISTRY_PATH = ErrorException(
+    "No path to the registry has been set. Run "*
+        "`setpath(path)`, where `path` is the path to the registry in your local "*
+        "MLJModels.jl clone. That is, do something like "*
+        "`setpath(\"~/MyPkgs/MLJModels/registry\")`. "
+)
+
+# for accessing or changing the location of the registry (initially empty):
+function registry_path()
+    out = REGISTRY_PATH[]
+    isempty(out) && throw(ERR_REGISTRY_PATH)
+    return out
+end
+setpath(path) = (REGISTRY_PATH[] = expanduser(path))