Merge pull request #593 from JuliaAI/dev

For a 0.18 release

Author: ablaom

.github/workflows/check_registry.yml

@@ -41,6 +41,9 @@ jobs:
             ${{ runner.os }}-
       - uses: julia-actions/julia-buildpkg@v1
       - uses: julia-actions/julia-runtest@v1
+        env:
+          # This environment variable enables the integration tests:
+          MLJ_TEST_REGISTRY: '1'
       - uses: julia-actions/julia-processcoverage@v1
       - uses: codecov/codecov-action@v4
         with:

.github/workflows/ci.yml

@@ -1,7 +1,7 @@
 name = "MLJModels"
 uuid = "d491faf4-2d78-11e9-2867-c94bc002c0b7"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.17.9"
+version = "0.18.0"
 
 [deps]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"
@@ -31,33 +31,39 @@ Tables = "bd369af6-aec1-5ad0-b16a-f7cc5008161c"
 CategoricalArrays = "0.9, 0.10"
 CategoricalDistributions = "0.1"
 Combinatorics = "1.0"
-Dates = "<0.0.1, 1"
+Dates = "1"
 Distances = "0.9,0.10"
+Distributed = "1"
 Distributions = "0.25"
-InteractiveUtils = "<0.0.1, 1"
-LinearAlgebra = "<0.0.1, 1"
-Markdown = "<0.0.1, 1"
+InteractiveUtils = "1"
+LinearAlgebra = "1"
 MLJModelInterface = "1.10"
+MLJTransforms = "0.1.1"
+Markdown = "1"
 OrderedCollections = "1.1"
 Parameters = "0.12"
-Pkg = "<0.0.1, 1"
+Pkg = "1"
 PrettyPrinting = "0.3, 0.4"
-Random = "<0.0.1, 1"
+Random = "1"
 RelocatableFolders = "0.3, 1"
 ScientificTypes = "3"
 StatisticalTraits = "3"
-Statistics = "<0.0.1, 1"
+Statistics = "1"
 StatsBase = "0.32,0.33, 0.34"
+Suppressor = "0.2.8"
 Tables = "0.2,1.0"
-julia = "1.6"
+julia = "1.10"
 
 [extras]
+Distributed = "8ba89e20-285c-5b6f-9357-94700520ee1b"
 MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
 MLJDecisionTreeInterface = "c6f25543-311c-4c74-83dc-3ea6d1015661"
 MLJMultivariateStatsInterface = "1b6a4a23-ba22-4f51-9698-8599985d3728"
+MLJTransforms = "23777cdb-d90c-4eb0-a694-7c2b83d5c1d6"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 StableRNGs = "860ef19b-820b-49d6-a774-d7a799459cd3"
+Suppressor = "fd094767-a336-5f1f-9728-57cf17d0bbfb"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["MLJBase", "MLJDecisionTreeInterface", "MLJMultivariateStatsInterface", "Pkg", "StableRNGs", "Test"]
+test = ["Distributed", "MLJBase", "MLJDecisionTreeInterface", "MLJMultivariateStatsInterface", "MLJTransforms", "Pkg", "StableRNGs", "Suppressor", "Test"]

Project.toml

@@ -1,79 +1,76 @@
-# MLJModels
+# MLJModels.jl
 
 [![Build Status](https://github.com/JuliaAI/MLJModels.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/MLJModels.jl/actions)
 [![codecov](https://codecov.io/gh/JuliaAI/MLJModels.jl/graph/badge.svg?token=KgarnnCc0K)](https://codecov.io/gh/JuliaAI/MLJModels.jl)
 
-Repository of the "built-in" models available for use in the
-[MLJ](https://github.com/JuliaAI/MLJ.jl) MLJ machine
-learning framework; and the home of the MLJ model registry.
+Home of the [MLJ](https://juliaml.ai) Model Registry and tools for model search and model code loading. 
 
-For instructions on integrating a new model with MLJ visit
-[here](https://JuliaAI.github.io/MLJ.jl/dev/adding_models_for_general_use/)
+For instructions on integrating a new model into MLJ visit
+[here](https://juliaai.github.io/MLJModelInterface.jl/stable/).
 
 
 ### Contents
 
  - [Who is this repo for?](#who-is-this-repo-for)
+ - [How to register new models](#how-to-register-new-models)
  - [What is provided here?](#what-is-provided-here)
- - [Instructions for updating the MLJ model registry](#instructions-for-updating-the-mlj-model-registry)
 
 ## Who is this repo for?
 
-General users of the MLJ machine learning platform should refer to
-[MLJ home page](https://JuliaAI.github.io/MLJ.jl/dev/)
-for usage and installation instructions. MLJModels is a dependency of
-MLJ that the general user can ignore.
+Newcomers to MLJ should refer to [this page](https://juliaml.ai) for usage and
+installation instructions. MLJModels.jl is a dependency of MLJ that the general user can
+ignore.
 
-This repository is for developers wishing to
-[register](#instructions-for-updating-the-mlj-model-registry) new MLJ
-model interfaces, whether they be:
+This repository is for developers maintaining:
 
-- implemented **natively** in a
-  package providing the core machine learning algorithm, as in
+- The [MLJ Model Registry](/src/registry), a database of packages implementing the MLJ
+  interface for machine learning models, together with metadata about those models.
+
+- MLJ tools for searching the database (`models(...)` and `matching(...)`) and for loading
+  model code (`@load`, `@iload`).
+
+## How to register new models
+
+The model registry lives at "/src/registry" but
+is maintained using
+[MLJModelRegistryTools.jl](https://juliaai.github.io/MLJModelRegistryTools.jl/dev/).
+
+New MLJ model interfaces can be implemented either:
+
+- **natively** in a package providing the core machine learning algorithm, as in
   [`EvoTrees.jl`](https://github.com/Evovest/EvoTrees.jl/blob/master/src/MLJ.jl); or
 
-- implemented in a separate **interface package**, such as
+-  in a separate **interface package**, such as
   [MLJDecisionTreeInterface.jl](https://github.com/JuliaAI/MLJDecisionTreeInterface.jl).
 
-It also a place for developers to add models (mostly transformers)
-such as `OneHotEncoder`, that are exported for "built-in" use in
-MLJ. (In the future these models may live in a separate package.)
-
-To list *all* model interfaces currently registered, do `using MLJ` or
-`using MLJModels` and run:
+In either case, the package providing the implementation needs to be added to the MLJ
+Model Registry to make it discoverable by MLJ users, and to make the model metadata
+searchable. To register a package, prepare a pull request to MLJModels.jl by following [these instructions](https://juliaai.github.io/MLJModelRegistryTools.jl/dev/registry_management_tools/#Registry-management-tools).
 
-- `localmodels()` to list built-in models (updated when external models are loaded with `@load`)
+Currently, after registering the model, one must also make a PR to MLJ updating [this
+dictionary of model
+descriptors](https://github.com/JuliaAI/MLJ.jl/blob/dev/docs/ModelDescriptors.toml) to
+ensure the new models appear in the right places in MLJ's [Model
+Browser](https://JuliaAI.github.io/MLJ.jl/dev/model_browser/#Model-Browser)
 
-- `models()` to list all registered models, or see [this list](/src/registry/Models.toml).
+To list *all* model interfaces currently registered, do `using MLJ` or `using MLJModels`
+and run `models()` to list all registered models.
 
 Recall that an interface is loaded from within MLJ, together with the
 package providing the underlying algorithm, using the syntax `@load
 RidgeRegressor pkg=GLM`, where the `pkg` keyword is only necessary in
 ambiguous cases.
 
-
 ## What is provided here?
 
-MLJModels contains:
-
-- transformers to be pre-loaded into MLJ, located at
-  [/src/builtins](/src/builtins), such as `OneHotEncoder`
-  and `ConstantClassifier`. 
-
-- the MLJ [model registry](src/registry/Metadata.toml), listing all
-  models that can be called from MLJ using `@load`. Package developers
-  can register new models by implementing the MLJ interface in their
-  package and following [these
-  instructions](https://JuliaAI.github.io/MLJ.jl/dev/adding_models_for_general_use/).
-  
-
-## Instructions for updating the MLJ model registry
+The actual MLJ Model Registry consists of the TOML files in [this
+directory](/src/registry). A few models available for immediate use in MLJ (without
+loading model code using `@load`) are also provided by this package, under "/src/builtins"
+but these may be moved out in the future.
 
-Generally model registration is performed by administrators. If you
-have an interface you would like registered, open an issue
-[here](https://github.com/JuliaAI/MLJ.jl/issues). 
+### Historical note
 
-**Administrator instructions.** These are given in the
-`MLJModels.@update` document string. After registering the model, make a PR to MLJ 
-updating [this dictionary of model descriptors](https://github.com/JuliaAI/MLJ.jl/blob/dev/docs/ModelDescriptors.toml) 
-to ensure the new models appear in the right places in MLJ's [Model Browser](https://JuliaAI.github.io/MLJ.jl/dev/model_browser/#Model-Browser)
+Older versions of MLJModels.jl contained some of the models now residing at
+[MLJTransforms.jl](https://github.com/JuliaAI/MLJTransforms.jl/tree/dev). Even older
+versions provided implementations of all the non-native implementations of the
+MLJ interface.

README.md

@@ -48,7 +48,6 @@ export UnivariateDiscretizer,
     OneHotEncoder, ContinuousEncoder, FillImputer, UnivariateFillImputer,
     UnivariateTimeTypeToContinuous, InteractionTransformer
 
-const srcdir = dirname(@__FILE__) # the directory containing this file
 const MMI = MLJModelInterface
 
 if VERSION < v"1.3"
@@ -61,31 +60,31 @@ include("utilities.jl")
 
 # load built-in models:
 include("builtins/Constant.jl")
-include("builtins/Transformers.jl")
 include("builtins/ThresholdPredictors.jl")
 
-Handle = NamedTuple{(:name, :pkg), Tuple{String,String}}
-(::Type{Handle})(name,string) = NamedTuple{(:name, :pkg)}((name, string))
+# declare paths to the metadata and associated project file:
+const REGISTRY_PROJECT = @path joinpath(@__DIR__, "registry", "Project.toml")
+const REGISTRY_METADATA = @path joinpath(@__DIR__, "registry", "Metadata.toml")
+Base.include_dependency(REGISTRY_PROJECT)
+Base.include_dependency(REGISTRY_METADATA)
 
 # load utilities for reading model metadata from file:
 include("metadata.jl")
 
-# read in the metadata:
-metadata_file = joinpath(srcdir, "registry", "Metadata.toml")
-Base.include_dependency(metadata_file)
-const INFO_GIVEN_HANDLE = info_given_handle(metadata_file)
+# read in metadata:
+const INFO_GIVEN_HANDLE = info_given_handle(REGISTRY_METADATA)
 const PKGS_GIVEN_NAME = pkgs_given_name(INFO_GIVEN_HANDLE)
 const AMBIGUOUS_NAMES = ambiguous_names(INFO_GIVEN_HANDLE)
 const NAMES = model_names(INFO_GIVEN_HANDLE)
 const MODEL_TRAITS_IN_REGISTRY = model_traits_in_registry(INFO_GIVEN_HANDLE)
 
-# model search and registry code:
+# include tools to search the model registry:
 include("model_search.jl")
+
+# include tools to load model code:
 include("loading.jl")
-include("registry/src/Registry.jl")
-using .Registry
 
-# finalize:
-include("init.jl")
+# include tool for cloning the Model Registry project file:
+include("registry_project.jl")
 
 end # module