Merge pull request #50 from JuliaAI/levels

Extend support for MLJ 0.22

Author: ablaom

Project.toml

@@ -1,7 +1,7 @@
 name = "MLJTestIntegration"
 uuid = "697918b4-fdc1-4f9e-8ff9-929724cee270"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.5.3"
+version = "0.5.4"
 
 [deps]
 MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"
@@ -12,7 +12,7 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
-MLJ = "0.18, 0.19, 0.20, 0.21"
+MLJ = "0.18, 0.19, 0.20, 0.21, 0.22"
 MLJModels = "0.16, 0.17, 0.18"
 MLJTestInterface = "0.2.4"
 MLJTransforms = "0.1.3"

README.md

@@ -4,10 +4,13 @@ Package for applying integration tests to models implementing the
 [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) model
 interface.
 
-**To test implementations of the MLJ model interface, use [MLJTestInterface.jl](https://github.com/JuliaAI/MLJTestInterface.jl)
-instead.**
+For to apply minimal tests for an implementations of the MLJ model interface, use
+[MLJTestInterface.jl](https://github.com/JuliaAI/MLJTestInterface.jl) instead.
 
-[![Lifecycle:Experimental](https://img.shields.io/badge/Lifecycle-Experimental-339999)](https://github.com/bcgov/repomountie/blob/master/doc/lifecycle-badges.md) [![Build Status](https://github.com/JuliaAI/MLJTestIntegration.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/MLJTestIntegration.jl/actions) [![Coverage](https://codecov.io/gh/JuliaAI/MLJTestIntegration.jl/branch/master/graph/badge.svg)](https://codecov.io/github/JuliaAI/MLJTestIntegration.jl?branch=master) 
+However, *implementations are now encouraged to include the comprehensive tests* provided
+in this package instead.
+
+[![Lifecycle:Experimental](https://img.shields.io/badge/Lifecycle-Experimental-339999)](https://github.com/bcgov/repomountie/blob/master/doc/lifecycle-badges.md) [![Build Status](https://github.com/JuliaAI/MLJTestIntegration.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/MLJTestIntegration.jl/actions) [![Coverage](https://codecov.io/gh/JuliaAI/MLJTestIntegration.jl/branch/master/graph/badge.svg)](https://codecov.io/github/JuliaAI/MLJTestIntegration.jl?branch=master)
 
 # Installation
 
@@ -19,55 +22,55 @@ Pkg.add("MLJTestIntegration")
 # Usage
 
 This package provides a method for testing a collection of `models`
-(types or named tuples with keys `:name` and `:package_name`) using
-the specified training `data`:
+(more, specifically model types) using the specified training `data`:
 
 ```julia
-MLJTestIntegration.test(models, data...; mod=Main, level=2, throw=false, verbosity=1) 
+MLJTestIntegration.test(models, data...; mod=Main, level=2, throw=false, verbosity=1)
     -> failures, summary
 ```
 
+For pure Julia models, `level=4` (the maximimum) is recommended. Otherwise `level=3` is
+recommended.
+
 For detailed documentation, run `using MLJTestIntegration; @doc MLJTestIntegration.test`.
 
 
-# Example: Testing models filtered from the MLJ model registry
+# Example
 
-The following applies comprehensive integration tests to all
-regressors provided by the package GLM.jl appearing in the MLJ Model
-Registry. Since GLM.jl models are provided through the interface
-package `MLJGLMInterface`, this must be in the current environment:
+The following tests the model interface implemented by the `DecisionTreeClassifier` model
+implemented in the package MLJDecisionTreeInterface.jl.
 
 ```julia
-Pkg.add("MLJGLMInterface")
-import MLJBase, MLJTestIntegration
-using DataFrames # to view summary
-X, y = MLJTestIntegration.MLJ.make_regression();
-regressors = MLJTestIntegration.MLJ.models(matching(X, y)) do m
-    m.package_name == "GLM"
-end
-
-# to test code loading:
-failures, summary = 
-    MLJTestIntegration.test(regressors, X, y, verbosity=2, mod=@__MODULE__, level=1)
-@assert isempty(failures)
-
-# comprehensive tests:
-failures, summary =
-    MLJTestIntegration.test(regressors, X, y, verbosity=2, mod=@__MODULE__, level=4)
-
-summary |> DataFrame
+import MLJDecisionTreeInterface
+import MLJTestIntegration
+using Test
+X, y = MLJTestIntegration.make_multiclass()
+failures, summary = MLJTestIntegration.test(
+    [MLJDecisionTreeInterface.DecisionTreeClassifier, ],
+    X, y,
+    verbosity=0, # set to 2 when debugging
+    throw=false, # set to `true` when debugging
+    mod=@__MODULE__,
+)
+@test isempty(failures)
 ```
 
+For models supporting tabular data `X`, we recommend repeating the tests with `(X, y)`
+replaced with `(Tables.rowtable(X)), y)` to ensure support for different table types,
+which new implementation sometimes fail to do.
+
 # Datasets
 
 The following commands generate datasets of the form `(X, y)` suitable for integration
 tests:
 
-- `MLJTestIntegration.make_binary` 
+- `MLJTestIntegration.make_binary`
 
-- `MLJTestIntegration.make_multiclass` 
+- `MLJTestIntegration.make_multiclass`
 
-- `MLJTestIntegration.make_regression` 
+- `MLJTestIntegration.make_regression`
 
-- `MLJTestIntegration.make_count` 
+- `MLJTestIntegration.make_count`
 
+The binary and multiclass sets are deliberately small and will catch failures to properly
+track target class pools, a common gotcha in new implementations.