diff --git a/Project.toml b/Project.toml index 3888153..9bf5739 100644 --- a/Project.toml +++ b/Project.toml @@ -1,7 +1,7 @@ name = "MLJModelRegistryTools" uuid = "0a96183e-380b-4aa6-be10-c555140810f2" authors = ["Anthony D. Blaom "] -version = "0.1.0" +version = "0.1.1" [deps] Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b" diff --git a/src/methods.jl b/src/methods.jl index 719db03..ee78438 100644 --- a/src/methods.jl +++ b/src/methods.jl @@ -11,8 +11,8 @@ err_invalid_packages(skip, env) = ArgumentError( "\"$env\". " ) -const INFO_BE_PATIENT1 = "Be patient. This could take a minute or so... " -const INFO_BE_PATIENT10 = "Be patient. This could take ten minutes or so..." +const INFO_BE_PATIENT1 = "Be patient. This could take a minute or so ... " +const INFO_BE_PATIENT10 = "Be patient. This could take ten minutes or so ..." # # HELPERS @@ -28,9 +28,8 @@ end # develop MLJModelRegistryTools into the specifified `registry` project: function setup(registry) ex = quote - # TODO: replace Line 1 with Line 2 after MLJModelRegistry is registered at General: - Pkg.develop(path=$ROOT) # Line 1 - # Pkg.add(MLJModelRegistryTools) # Line 2 + # Pkg.develop(path=$ROOT) + Pkg.add("MLJModelRegistryTools") end future = GenericRegistry.run([], ex; environment=registry) fetch(future) @@ -66,9 +65,8 @@ function metadata(pkg; registry="", check_traits=true) setup=() else setup = quote - # REMOVE THIS NEXT LINE AFTER TAGGING NEW MLJMODELINTERFACE - Pkg.develop(path="/Users/anthony/MLJ/MLJModelInterface/") - Pkg.develop(path=$ROOT) # MLJModelRegistryTools + # Pkg.develop(path=$ROOT) + Pkg.add("MLJModelRegistryTools") end end program = quote @@ -194,18 +192,23 @@ function update( issubset(skip, allpkgs) || throw(err_invalid_packages(skip, registry)) @suppress setup(registry) end - pkgs = setdiff(allpkgs, skip) - N = length(pkgs) - pos = 1 + pkgs = setdiff(allpkgs, skip) |> sort + pkg_set = OrderedSet(pkgs) + @info "Processing up to $nworkers packages at a time. " @info INFO_BE_PATIENT10 - while N ≥ 1 - print("\rPackages remaining: $N ") - n = min(nworkers, N) - batch = pkgs[pos:pos + n - 1] + while !isempty(pkg_set) + print("\rPackages remaining: $(length(pkgs)) ") + n = min(nworkers, length(pkg_set)) + batch = [pop!(pkg_set) for _ in 1:n] @suppress begin - futures = - [MLJModelRegistryTools.metadata(pkg; registry, check_traits) for pkg in batch] + futures = [ + MLJModelRegistryTools.metadata( + pkg; + registry, + check_traits, + ) for pkg in batch + ] try for (i, f) in enumerate(futures) GenericRegistry.put(batch[i], fetch(f), registry_path()) @@ -217,7 +220,6 @@ function update( end debug || GenericRegistry.close.(futures) end - N -= n end isempty(registry) || @suppress cleanup(registry) gc() diff --git a/src/remote_methods.jl b/src/remote_methods.jl index 72b5960..ea4f949 100644 --- a/src/remote_methods.jl +++ b/src/remote_methods.jl @@ -51,7 +51,8 @@ for converting leaves are: 2. If it's an `AbstractString`, apply `string` function (e.g, to remove `SubString`s). -3. In all other cases, except `AbstractArray`s, wrap in single quotes, as in sum -> "`sum`". +3. In all other cases, except `AbstractArray`s, first wrap in single quotes, as in sum -> "\`sum\`". + 4. Replace any `#` character in the application of Rule 3 with `_` (to handle `gensym` names) 5. For an `AbstractVector`, broadcast the preceding Rules over its elements. diff --git a/src/setpath.jl b/src/setpath.jl index fa62769..2df9fc3 100644 --- a/src/setpath.jl +++ b/src/setpath.jl @@ -4,7 +4,7 @@ const ERR_REGISTRY_PATH = ErrorException( "No path to the registry has been set. Run "* "`setpath(path)`, where `path` is the path to the registry in your local "* "MLJModels.jl clone. That is, do something like "* - "`setpath(\"~/MyPkgs/MLJModels/registry\")`. " + "`setpath(\"~/MyPkgs/MLJModels/src/registry\")`. " ) # for accessing or changing the location of the registry (initially empty):