Merge pull request #8 from JuliaAI/dev

For a 0.1.0 release (second attempt after repository name change)

Author: ablaom

Project.toml

@@ -1,4 +1,4 @@
-name = "MLJTest"
+name = "MLJTestIntegration"
 uuid = "697918b4-fdc1-4f9e-8ff9-929724cee270"
 authors = ["Anthony D. Blaom <[email protected]>"]
 version = "0.1.0"
@@ -9,4 +9,4 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 
 [compat]
 MLJ = "0.18"
-julia = "1.6"
+julia = "1.6"

README.md

@@ -1,16 +1,16 @@
-# MLJTest.jl
+# MLJTestIntegration.jl
 
 Package for applying integration tests to models implementing the
 [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) model
 interface.
 
-[![Lifecycle:Experimental](https://img.shields.io/badge/Lifecycle-Experimental-339999)](https://github.com/bcgov/repomountie/blob/master/doc/lifecycle-badges.md) [![Build Status](https://github.com/JuliaAI/MLJTest.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/MLJTest.jl/actions) [![Coverage](https://codecov.io/gh/JuliaAI/MLJTest.jl/branch/master/graph/badge.svg)](https://codecov.io/github/JuliaAI/MLJTest.jl?branch=master) 
+[![Lifecycle:Experimental](https://img.shields.io/badge/Lifecycle-Experimental-339999)](https://github.com/bcgov/repomountie/blob/master/doc/lifecycle-badges.md) [![Build Status](https://github.com/JuliaAI/MLJTestIntegration.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/MLJTestIntegration.jl/actions) [![Coverage](https://codecov.io/gh/JuliaAI/MLJTestIntegration.jl/branch/master/graph/badge.svg)](https://codecov.io/github/JuliaAI/MLJTestIntegration.jl?branch=master) 
 
 # Installation
 
 ```julia
 using Pkg
-Pkg.add("MLJTest")
+Pkg.add("MLJTestIntegration")
 ```
 
 # Usage
@@ -19,8 +19,52 @@ This package provides a single method for testing a collection of
 `models` (types or named tuples with keys `:name` and `:package_name`)
 using the specified training `data`:
 
+```julia
+MLJTestIntegration.test(models, data...; mod=Main, level=2, throw=false, verbosity=1) 
+    -> failures, summary
 ```
-MLJTest.test(models, data...; verbosity=1, mod=Main, loading_only=false) -> failures, summary
+
+For detailed documentation, run `using MLJTestIntegration; @doc MLJTestIntegration.test`.
+
+
+# Examples
+
+## Testing models in a new MLJ model interface implementation
+
+The following tests the model interface implemented by some model type
+`MyClassifier`, as might appear in tests for a package providing that
+type:
+
+```julia
+import MLJTestIntegration
+using Test
+X, y = MLJTestIntegration.MLJ.make_blobs()
+failures, summary = MLJTestIntegration.test([MyClassifier, ], X, y, verbosity=1, mod=@__MODULE__)
+@test isempty(failures)
 ```
 
-See the method document string for details. 
+## Testing models after filtering models in the registry
+
+The following applies comprehensive integration tests to all
+regressors provided by the package GLM.jl appearing in the MLJ Model
+Registry. Since GLM.jl models are provided through the interface
+package `MLJGLMInterface`, this must be in the current environment:
+
+```julia
+Pkg.add("MLJGLMInterface")
+import MLJBase, MLJTestIntegration
+using DataFrames # to view summary
+X, y = MLJTestIntegration.MLJ.make_regression();
+regressors = MLJTestIntegration.MLJ.models(matching(X, y)) do m
+    m.package_name == "GLM"
+end
+
+# to test code loading *and* load code:
+MLJTestIntegration.test(regressors, X, y, verbosity=1, mod=@__MODULE__, level=1)
+
+# comprehensive tests:
+failures, summary =
+    MLJTestIntegration.test(regressors, X, y, verbosity=3, mod=@__MODULE__, level=1)
+
+summary |> DataFrame
+```

examples/bigtest/Project.toml

@@ -0,0 +1,33 @@
+[deps]
+BetaML = "024491cd-cc6b-443e-8034-08ea7eb7db2b"
+DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
+EvoTrees = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5"
+InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
+LightGBM = "7acf609c-83a4-11e9-1ffb-b912bcd3b04a"
+MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
+MLJClusteringInterface = "d354fa79-ed1c-40d4-88ef-b8c7bd1568af"
+MLJDecisionTreeInterface = "c6f25543-311c-4c74-83dc-3ea6d1015661"
+MLJFlux = "094fc8d1-fd35-5302-93ea-dabda2abf845"
+MLJGLMInterface = "caf8df21-4939-456d-ac9c-5fefbfb04c0c"
+MLJLIBSVMInterface = "61c7150f-6c77-4bb1-949c-13197eac2a52"
+MLJLinearModels = "6ee0df7b-362f-4a72-a706-9e79364fb692"
+MLJModels = "d491faf4-2d78-11e9-2867-c94bc002c0b7"
+MLJMultivariateStatsInterface = "1b6a4a23-ba22-4f51-9698-8599985d3728"
+MLJNaiveBayesInterface = "33e4bacb-b9e2-458e-9a13-5d9a90b235fa"
+MLJScikitLearnInterface = "5ae90465-5518-4432-b9d2-8a1def2f0cab"
+MLJTSVDInterface = "7fa162e1-0e29-41ca-a6fa-c000ca4e7e7e"
+MLJTestIntegration = "697918b4-fdc1-4f9e-8ff9-929724cee270"
+MLJText = "5e27fcf9-6bac-46ba-8580-b5712f3d6387"
+MLJXGBoostInterface = "54119dfa-1dab-4055-a167-80440f4f7a91"
+NearestNeighborModels = "636a865e-7cf4-491e-846c-de09b730eb36"
+OneRule = "90484964-6d6a-4979-af09-8657dbed84ff"
+OutlierDetectionNeighbors = "51249a0a-cb36-4849-8e04-30c7f8d311bb"
+OutlierDetectionNetworks = "c7f57e37-4fcb-4a0b-a36c-c2204bc839a7"
+OutlierDetectionPython = "2449c660-d36c-460e-a68b-92ab3c865b3e"
+ParallelKMeans = "42b8e9d4-006b-409a-8472-7f34b3fb58af"
+PartialLeastSquaresRegressor = "f4b1acfe-f311-436c-bb79-8483f53c17d5"
+Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

examples/bigtest/notebook.jl

@@ -0,0 +1,51 @@
+# **Assumption.** This file has not been separated from the
+# Project.toml file that
+# [originally](https://github.com/JuliaAI/MLJTestIntegration.jl/blob/dev/examples/bigtest/Project.toml)
+# accompanied it.
+
+using Pkg
+Pkg.activate(@__DIR__)
+Pkg.instantiate()
+
+using MLJTestIntegration
+using MLJModels
+using Test
+using DataFrames # for displaying tables
+
+if false
+# # Regression
+
+known_issues = models() do model
+    any([
+        # https://github.com/lalvim/PartialLeastSquaresRegressor.jl/issues/29
+        model.package_name == "PartialLeastSquaresRegressor",
+
+        #https://github.com/JuliaAI/MLJXGBoostInterface.jl/issues/17
+        model.name == "XGBoostRegressor",
+    ])
+end
+
+MLJTestIntegration.test_single_target_regressors(ignore=known_issues, level=1)
+fails, summary =
+    MLJTestIntegration.test_single_target_regressors(ignore=known_issues, level=3)
+
+@test isempty(fails)
+summary |> DataFrame
+end
+
+
+# # Classification
+
+# https://github.com/alan-turing-institute/MLJ.jl/issues/939
+known_issues = [
+    #
+    (name = "DecisionTreeClassifier", package_name="BetaML"),
+    (name = "NuSVC", package_name="LIBSVM"),
+    (name="PegasosClassifier", package_name="BetaML"),
+    (name="RandomForestClassifier", package_name="BetaML"),
+    (name="SVMNuClassifier", package_name="ScikitLearn"),
+]
+
+MLJTestIntegration.test_single_target_classifiers(ignore=known_issues, level=1)
+fails, summary =
+    MLJTestIntegration.test_single_target_classifiers(ignore=known_issues, level=3)

src/MLJTest.jl renamed to src/MLJTestIntegration.jl

@@ -1,10 +1,11 @@
-module MLJTest
+module MLJTestIntegration
 
 using MLJ
 using Pkg
 
 include("attemptors.jl")
 include("test.jl")
+include("special_cases.jl")
 include("dummy_model.jl")
 
 using .DummyModel

src/attemptors.jl

@@ -1,20 +1,20 @@
-str(model_metadata) = "$(model_metadata.name) from $(model_metadata.package_name)"
-
 """
-    attempt(f, message="")
+    attempt(f, message; throw=false)
 
 Return `(f(), "✓") if `f()` executes without throwing an
 exception. Otherwise, return `(ex, "×"), where `ex` is the exception
-thrown.
+caught. Only truly throw the exception if `throw=true`. 
 
 If `message` is not empty, then it is logged to `Info`, together with
 the second return value ("✓" or "×").
 
+
 """
-function attempt(f, message="")
+function attempt(f, message; throw=false)
     ret = try
         (f(), "✓")
     catch ex
+        throw && Base.throw(ex)
         (ex, "×")
     end
     isempty(message) || @info message*last(ret)
@@ -30,7 +30,7 @@ finalize(message, verbosity) = verbosity < 2 ? "" : message
 # is updated to 0.16. And delete the two methods immediately
 # following. What's required will already be in MLJModels 0.15.10, but
 # the current implementation avoids an explicit MLJModels dependency
-# for MLJTest.
+# for MLJTestIntegration.
 load_path(model_type) = MLJ.load_path(model_type)
 function load_path(proxy::NamedTuple)
     handle = (name=proxy.name, pkg=proxy.package_name)
@@ -39,11 +39,11 @@ end
 
 root(load_path) = split(load_path, '.') |> first
 
-function model_type(proxy, mod; verbosity=1)
+function model_type(proxy, mod; throw=false, verbosity=1)
     # check interface package really is in current environment:
-    message = "`[:model_type]` Loading model type "
-    model_type, outcome = attempt(finalize(message, verbosity)) do
-        load_path = MLJTest.load_path(proxy) # MLJ.load_path(proxy) *****
+    message = "[:model_type] Loading model type "
+    model_type, outcome = attempt(finalize(message, verbosity); throw) do
+        load_path = MLJTestIntegration.load_path(proxy) # MLJ.load_path(proxy) *****
         load_path_ex = load_path |> Meta.parse
         api_pkg_ex = root(load_path) |> Symbol
         import_ex = :(import $api_pkg_ex)
@@ -61,39 +61,39 @@ function model_type(proxy, mod; verbosity=1)
         # above, `model_type`, was triggered because of API package is
         # missing from in environment.
         api_pkg = try
-            load_path = MLJTest.load_path(proxy) # MLJ.load_path(proxy) *****
+            load_path = MLJTestIntegration.load_path(proxy) # MLJ.load_path(proxy) *****
             api_pkg = root(load_path)
         catch
             nothing
         end
         if !isnothing(api_pkg) &&
                api_pkg != "unknown" &&
                contains(model_type.msg, "$api_pkg not found in")
-            throw(model_type)
+            Base.throw(model_type)
         end
     end
 
     return model_type, outcome
 end
 
-function model_instance(model_type; verbosity=1)
-    message = "`[:model_instance]` Instantiating default model "
-    attempt(finalize(message, verbosity))  do
+function model_instance(model_type; throw=false, verbosity=1)
+    message = "[:model_instance] Instantiating default model "
+    attempt(finalize(message, verbosity); throw)  do
         model_type()
     end
 end
 
-function fitted_machine(model, data...; verbosity=1)
-    message = "`[:fitted_machine]` Fitting machine "
-    attempt(finalize(message, verbosity))  do
+function fitted_machine(model, data...; throw=false, verbosity=1)
+    message = "[:fitted_machine] Fitting machine "
+    attempt(finalize(message, verbosity); throw)  do
         mach = machine(model, data...)
         fit!(mach, verbosity=-1)
     end
 end
 
-function operations(fitted_machine, data...; verbosity=1)
-    message = "`[:operations]` Calling `predict`, `transform` and/or `inverse_transform` "
-    attempt(finalize(message, verbosity))  do
+function operations(fitted_machine, data...; throw=false, verbosity=1)
+    message = "[:operations] Calling `predict`, `transform` and/or `inverse_transform` "
+    attempt(finalize(message, verbosity); throw)  do
         operations = String[]
         methods = MLJ.implemented_methods(fitted_machine.model)
         if :predict in methods
@@ -112,30 +112,30 @@ function operations(fitted_machine, data...; verbosity=1)
     end
 end
 
-function threshold_prediction(model, data...; verbosity=1)
-    message = "`[:threshold_predictor]` Calling fit!/predict for threshold predictor "*
+function threshold_prediction(model, data...; throw=false, verbosity=1)
+    message = "[:threshold_predictor] Calling fit!/predict for threshold predictor "*
         "test) "
-    attempt(finalize(message, verbosity)) do
+    attempt(finalize(message, verbosity); throw) do
         tmodel = BinaryThresholdPredictor(model)
         mach = machine(tmodel, data...)
         fit!(mach, verbosity=0)
         predict(mach, first(data))
     end
 end
 
-function evaluation(measure, model, data...; verbosity=1)
-    message = "`[:evaluation]` Evaluating performance "
-    attempt(finalize(message, verbosity)) do
+function evaluation(measure, model, data...; throw=false, verbosity=1)
+    message = "[:evaluation] Evaluating performance "
+    attempt(finalize(message, verbosity); throw) do
         evaluate(model, data...;
                  measure=measure,
                  resampling=Holdout(),
                  verbosity=0)
     end
 end
 
-function tuned_pipe_evaluation(measure, model, data...; verbosity=1)
-    message = "`[:tuned_pipe_evaluation]` Evaluating perfomance in a tuned pipeline "
-    attempt(finalize(message, verbosity)) do
+function tuned_pipe_evaluation(measure, model, data...; throw=false, verbosity=1)
+    message = "[:tuned_pipe_evaluation] Evaluating perfomance in a tuned pipeline "
+    attempt(finalize(message, verbosity); throw) do
         pipe = identity |> model
         tuned_pipe = TunedModel(models=[pipe,],
                                 measure=measure)
@@ -145,8 +145,8 @@ function tuned_pipe_evaluation(measure, model, data...; verbosity=1)
     end
 end
 
-function ensemble_prediction(model, data...; verbosity=1)
-    attempt(finalize("`[:ensemble_prediction]` Ensembling ", verbosity)) do
+function ensemble_prediction(model, data...; throw=false, verbosity=1)
+    attempt(finalize("[:ensemble_prediction] Ensembling ", verbosity); throw) do
         imodel = EnsembleModel(model=model,
                                n=2)
         mach = machine(imodel, data...)
@@ -155,9 +155,9 @@ function ensemble_prediction(model, data...; verbosity=1)
     end
 end
 
-function iteration_prediction(measure, model, data...; verbosity=1)
-    message =  "`[:iteration_prediction]` Iterating with controls "
-    attempt(finalize(message, verbosity)) do
+function iteration_prediction(measure, model, data...; throw=false, verbosity=1)
+    message =  "[:iteration_prediction] Iterating with controls "
+    attempt(finalize(message, verbosity); throw) do
         imodel = IteratedModel(model=model,
                                measure=measure,
                                controls=[Step(1),

src/dummy_model.jl

@@ -142,8 +142,8 @@ end
 MMI.predict(::DummyIterativeModel, fitresult, Xnew) =
     [fitresult[c].avg for c in Xnew]
 
-MMI.package_name(::Type{<:DummyIterativeModel}) = "MLJTest"
-MMI.load_path(::Type{<:DummyIterativeModel}) = "MLJTest.DummyIterativeModel"
+MMI.package_name(::Type{<:DummyIterativeModel}) = "MLJTestIntegration"
+MMI.load_path(::Type{<:DummyIterativeModel}) = "MLJTestIntegration.DummyIterativeModel"
 MMI.iteration_parameter(::Type{<:DummyIterativeModel}) = :n
 MMI.training_losses(::DummyIterativeModel, report) = report.training_losses