Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion Project.toml
Original file line number Diff line number Diff line change
@@ -1,7 +1,7 @@
name = "MLJModelRegistryTools"
uuid = "0a96183e-380b-4aa6-be10-c555140810f2"
authors = ["Anthony D. Blaom <[email protected]>"]
version = "0.1.0"
version = "0.1.1"

[deps]
Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
Expand Down
38 changes: 20 additions & 18 deletions src/methods.jl
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,8 @@ err_invalid_packages(skip, env) = ArgumentError(
"\"$env\". "
)

const INFO_BE_PATIENT1 = "Be patient. This could take a minute or so... "
const INFO_BE_PATIENT10 = "Be patient. This could take ten minutes or so..."
const INFO_BE_PATIENT1 = "Be patient. This could take a minute or so ... "
const INFO_BE_PATIENT10 = "Be patient. This could take ten minutes or so ..."


# # HELPERS
Expand All @@ -28,9 +28,8 @@ end
# develop MLJModelRegistryTools into the specifified `registry` project:
function setup(registry)
ex = quote
# TODO: replace Line 1 with Line 2 after MLJModelRegistry is registered at General:
Pkg.develop(path=$ROOT) # Line 1
# Pkg.add(MLJModelRegistryTools) # Line 2
# Pkg.develop(path=$ROOT)
Pkg.add("MLJModelRegistryTools")
end
future = GenericRegistry.run([], ex; environment=registry)
fetch(future)
Expand Down Expand Up @@ -66,9 +65,8 @@ function metadata(pkg; registry="", check_traits=true)
setup=()
else
setup = quote
# REMOVE THIS NEXT LINE AFTER TAGGING NEW MLJMODELINTERFACE
Pkg.develop(path="/Users/anthony/MLJ/MLJModelInterface/")
Pkg.develop(path=$ROOT) # MLJModelRegistryTools
# Pkg.develop(path=$ROOT)
Pkg.add("MLJModelRegistryTools")
end
end
program = quote
Expand Down Expand Up @@ -194,18 +192,23 @@ function update(
issubset(skip, allpkgs) || throw(err_invalid_packages(skip, registry))
@suppress setup(registry)
end
pkgs = setdiff(allpkgs, skip)
N = length(pkgs)
pos = 1
pkgs = setdiff(allpkgs, skip) |> sort
pkg_set = OrderedSet(pkgs)

@info "Processing up to $nworkers packages at a time. "
@info INFO_BE_PATIENT10
while N ≥ 1
print("\rPackages remaining: $N ")
n = min(nworkers, N)
batch = pkgs[pos:pos + n - 1]
while !isempty(pkg_set)
print("\rPackages remaining: $(length(pkgs)) ")
n = min(nworkers, length(pkg_set))
batch = [pop!(pkg_set) for _ in 1:n]
@suppress begin
futures =
[MLJModelRegistryTools.metadata(pkg; registry, check_traits) for pkg in batch]
futures = [
MLJModelRegistryTools.metadata(
pkg;
registry,
check_traits,
) for pkg in batch
]
try
for (i, f) in enumerate(futures)
GenericRegistry.put(batch[i], fetch(f), registry_path())
Expand All @@ -217,7 +220,6 @@ function update(
end
debug || GenericRegistry.close.(futures)
end
N -= n
end
isempty(registry) || @suppress cleanup(registry)
gc()
Expand Down
3 changes: 2 additions & 1 deletion src/remote_methods.jl
Original file line number Diff line number Diff line change
Expand Up @@ -51,7 +51,8 @@ for converting leaves are:

2. If it's an `AbstractString`, apply `string` function (e.g, to remove `SubString`s).

3. In all other cases, except `AbstractArray`s, wrap in single quotes, as in sum -> "`sum`".
3. In all other cases, except `AbstractArray`s, first wrap in single quotes, as in sum -> "\`sum\`".

4. Replace any `#` character in the application of Rule 3 with `_` (to handle `gensym` names)

5. For an `AbstractVector`, broadcast the preceding Rules over its elements.
Expand Down
2 changes: 1 addition & 1 deletion src/setpath.jl
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,7 @@ const ERR_REGISTRY_PATH = ErrorException(
"No path to the registry has been set. Run "*
"`setpath(path)`, where `path` is the path to the registry in your local "*
"MLJModels.jl clone. That is, do something like "*
"`setpath(\"~/MyPkgs/MLJModels/registry\")`. "
"`setpath(\"~/MyPkgs/MLJModels/src/registry\")`. "
)

# for accessing or changing the location of the registry (initially empty):
Expand Down
Loading