Merge pull request #51 from JuliaAI/dev

For a 0.5.4 release

Author: ablaom

.github/workflows/TagBot.yml

@@ -4,6 +4,9 @@ on:
     types:
       - created
   workflow_dispatch:
+    inputs:
+      lookback:
+        default: "3"
 jobs:
   TagBot:
     if: github.event_name == 'workflow_dispatch' || github.actor == 'JuliaTagBot'
@@ -12,4 +15,6 @@ jobs:
       - uses: JuliaRegistries/TagBot@v1
         with:
           token: ${{ secrets.GITHUB_TOKEN }}
+          # Edit the following line to reflect the actual name of the GitHub Secret containing your private key
           ssh: ${{ secrets.DOCUMENTER_KEY }}
+          # ssh: ${{ secrets.NAME_OF_MY_SSH_PRIVATE_KEY_SECRET }}

Project.toml

@@ -1,7 +1,7 @@
 name = "MLJTestIntegration"
 uuid = "697918b4-fdc1-4f9e-8ff9-929724cee270"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.5.3"
+version = "0.5.4"
 
 [deps]
 MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"
@@ -12,9 +12,10 @@ Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [compat]
-MLJ = "0.18, 0.19, 0.20, 0.21"
+MLJ = "0.18, 0.19, 0.20, 0.21, 0.22"
 MLJModels = "0.16, 0.17, 0.18"
 MLJTestInterface = "0.2.4"
+MLJTransforms = "0.1.3"
 NearestNeighborModels = "0.2"
 Pkg = "<0.0.1, 1"
 Test = "<0.0.1, 1"
@@ -23,7 +24,8 @@ julia = "1.10"
 [extras]
 MLJDecisionTreeInterface = "c6f25543-311c-4c74-83dc-3ea6d1015661"
 MLJModels = "d491faf4-2d78-11e9-2867-c94bc002c0b7"
+MLJTransforms = "23777cdb-d90c-4eb0-a694-7c2b83d5c1d6"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 
 [targets]
-test = ["MLJDecisionTreeInterface", "MLJModels", "Test"]
+test = ["MLJDecisionTreeInterface", "MLJModels", "MLJTransforms", "Test"]

README.md

@@ -4,10 +4,13 @@ Package for applying integration tests to models implementing the
 [MLJ](https://alan-turing-institute.github.io/MLJ.jl/dev/) model
 interface.
 
-**To test implementations of the MLJ model interface, use [MLJTestInterface.jl](https://github.com/JuliaAI/MLJTestInterface.jl)
-instead.**
+For to apply minimal tests for an implementations of the MLJ model interface, use
+[MLJTestInterface.jl](https://github.com/JuliaAI/MLJTestInterface.jl) instead.
 
-[![Lifecycle:Experimental](https://img.shields.io/badge/Lifecycle-Experimental-339999)](https://github.com/bcgov/repomountie/blob/master/doc/lifecycle-badges.md) [![Build Status](https://github.com/JuliaAI/MLJTestIntegration.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/MLJTestIntegration.jl/actions) [![Coverage](https://codecov.io/gh/JuliaAI/MLJTestIntegration.jl/branch/master/graph/badge.svg)](https://codecov.io/github/JuliaAI/MLJTestIntegration.jl?branch=master) 
+However, *implementations are now encouraged to include the comprehensive tests* provided
+in this package instead.
+
+[![Lifecycle:Experimental](https://img.shields.io/badge/Lifecycle-Experimental-339999)](https://github.com/bcgov/repomountie/blob/master/doc/lifecycle-badges.md) [![Build Status](https://github.com/JuliaAI/MLJTestIntegration.jl/workflows/CI/badge.svg)](https://github.com/JuliaAI/MLJTestIntegration.jl/actions) [![Coverage](https://codecov.io/gh/JuliaAI/MLJTestIntegration.jl/branch/master/graph/badge.svg)](https://codecov.io/github/JuliaAI/MLJTestIntegration.jl?branch=master)
 
 # Installation
 
@@ -19,55 +22,55 @@ Pkg.add("MLJTestIntegration")
 # Usage
 
 This package provides a method for testing a collection of `models`
-(types or named tuples with keys `:name` and `:package_name`) using
-the specified training `data`:
+(more, specifically model types) using the specified training `data`:
 
 ```julia
-MLJTestIntegration.test(models, data...; mod=Main, level=2, throw=false, verbosity=1) 
+MLJTestIntegration.test(models, data...; mod=Main, level=2, throw=false, verbosity=1)
     -> failures, summary
 ```
 
+For pure Julia models, `level=4` (the maximimum) is recommended. Otherwise `level=3` is
+recommended.
+
 For detailed documentation, run `using MLJTestIntegration; @doc MLJTestIntegration.test`.
 
 
-# Example: Testing models filtered from the MLJ model registry
+# Example
 
-The following applies comprehensive integration tests to all
-regressors provided by the package GLM.jl appearing in the MLJ Model
-Registry. Since GLM.jl models are provided through the interface
-package `MLJGLMInterface`, this must be in the current environment:
+The following tests the model interface implemented by the `DecisionTreeClassifier` model
+implemented in the package MLJDecisionTreeInterface.jl.
 
 ```julia
-Pkg.add("MLJGLMInterface")
-import MLJBase, MLJTestIntegration
-using DataFrames # to view summary
-X, y = MLJTestIntegration.MLJ.make_regression();
-regressors = MLJTestIntegration.MLJ.models(matching(X, y)) do m
-    m.package_name == "GLM"
-end
-
-# to test code loading:
-failures, summary = 
-    MLJTestIntegration.test(regressors, X, y, verbosity=2, mod=@__MODULE__, level=1)
-@assert isempty(failures)
-
-# comprehensive tests:
-failures, summary =
-    MLJTestIntegration.test(regressors, X, y, verbosity=2, mod=@__MODULE__, level=4)
-
-summary |> DataFrame
+import MLJDecisionTreeInterface
+import MLJTestIntegration
+using Test
+X, y = MLJTestIntegration.make_multiclass()
+failures, summary = MLJTestIntegration.test(
+    [MLJDecisionTreeInterface.DecisionTreeClassifier, ],
+    X, y,
+    verbosity=0, # set to 2 when debugging
+    throw=false, # set to `true` when debugging
+    mod=@__MODULE__,
+)
+@test isempty(failures)
 ```
 
+For models supporting tabular data `X`, we recommend repeating the tests with `(X, y)`
+replaced with `(Tables.rowtable(X)), y)` to ensure support for different table types,
+which new implementation sometimes fail to do.
+
 # Datasets
 
 The following commands generate datasets of the form `(X, y)` suitable for integration
 tests:
 
-- `MLJTestIntegration.make_binary` 
+- `MLJTestIntegration.make_binary`
 
-- `MLJTestIntegration.make_multiclass` 
+- `MLJTestIntegration.make_multiclass`
 
-- `MLJTestIntegration.make_regression` 
+- `MLJTestIntegration.make_regression`
 
-- `MLJTestIntegration.make_count` 
+- `MLJTestIntegration.make_count`
 
+The binary and multiclass sets are deliberately small and will catch failures to properly
+track target class pools, a common gotcha in new implementations.

test/datasets.jl

@@ -6,7 +6,7 @@ const MTI = MLJTestIntegration
 
 Rgs = @load KNNRegressor pkg=NearestNeighborModels verbosity=0
 Clf = @load KNNClassifier pkg= NearestNeighborModels verbosity=0
-Trf = @load Standardizer pkg=MLJModels verbosity=0
+Trf = @load Standardizer pkg=MLJTransforms verbosity=0
 
 Rgs_proxy = models() do m
     m.name == "KNNRegressor" && m.package_name == "NearestNeighborModels"