Merge pull request #27 from JuliaAI/trait-smoke-tests

Add trait smoke tests

Author: ablaom

.github/workflows/ci.yml

@@ -29,7 +29,7 @@ jobs:
         with:
           version: ${{ matrix.version }}
           arch: ${{ matrix.arch }}
-      - uses: actions/cache@v1
+      - uses: julia-actions/cache@v1
         env:
           cache-name: cache-artifacts
         with:

.github/workflows/ci_nightly.yml

@@ -1,7 +1,7 @@
 name = "MLJTestInterface"
 uuid = "72560011-54dd-4dc2-94f3-c5de45b75ecd"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.2.8"
+version = "0.2.9"
 
 [deps]
 MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"

Project.toml

@@ -65,6 +65,51 @@ function model_type(T, mod; throw=false, verbosity=1)
     return model_type, outcome
 end
 
+# helpers:
+ismissing_or_isa(x, T) = ismissing(x) || x isa T
+bad_trait(model_type) = "$model_type has a bad trait declaration.\n"
+
+const err_is_pure_julia(model_type) = ErrorException(
+    bad_trait(model_type)*"`is_pure_julia` must return `true` or `false`. "
+)
+const err_supports_weights(model_type) = ErrorException(
+    bad_trait(model_type)*"`supports_weights` must return `true`, `false` or `missing`. "
+)
+const err_supports_class_weights(model_type) = ErrorException(
+    bad_trait(model_type)*"`supports__class_weights` must return `true`, `false` or `missing`. "
+)
+const err_is_wrapper(model_type) = ErrorException(
+    bad_trait(model_type)*"`is_wrapper` must return `true` or `false`. "
+)
+const err_package_name(model_type) = ErrorException(
+    bad_trait(model_type)*"`package_name` must return a `String`. "
+)
+const err_packge_license(model_type) = ErrorException(
+    bad_trait(model_type)*"`package_license` must return a `String`. "
+)
+const err_iteration_parameter(model_type) = ErrorException(
+    bad_trait(model_type)*"`iteration_parameter` must return a `Symbol` or `nothing`. "
+)
+
+function traits(model_type; throw=false, verbosity=1)
+    message = "[:traits] Apply smoke test to some model traits"
+    attempt(finalize(message, verbosity); throw)  do
+        ismissing_or_isa(MLJBase.is_pure_julia(model_type), Bool) ||
+            throw(err_is_pure_julia(model_type))
+        ismissing_or_isa(MLJBase.supports_weights(model_type), Bool) ||
+            throw(err_supports_(model_type))
+        ismissing_or_isa(MLJBase.supports_class_weights(model_type), Bool) ||
+            throw(err_supports_class_weights(model_type))
+        MLJBase.package_name(model_type) isa String ||
+            throw(err_package_name(model_type))
+        MLJBase.package_license(model_type) isa String ||
+            throw(err_package_license(model_type))
+        MLJBase.iteration_parameter(model_type) isa Union{Nothing,Symbol} ||
+            throw(err_iteration_parameter(model_type))
+        nothing
+    end
+end
+
 function model_instance(model_type; throw=false, verbosity=1)
     message = "[:model_instance] Instantiating default model "
     attempt(finalize(message, verbosity); throw)  do

src/attemptors.jl

@@ -64,10 +64,10 @@ loaded into the module `mod`.
 
 The extent of testing is controlled by `level`:
 
-|`level`          | description                    | tests (full list below) |
-|:----------------|:-------------------------------|:------------------------|
-| 1               | test code loading              | `:model_type`           |
-| 2 (default)     | basic test of model interface  | all four tests          |
+|`level`          | description                    | tests (full list below)  |
+|:----------------|:-------------------------------|:-------------------------|
+| 1               | test code loading              | `:model_type`, `:traits` |
+| 2 (default)     | basic test of model interface  | all five tests           |
 
 For extensive MLJ integration tests, instead use `MLJTestIntegration.test`, from
 MLJTestIntegration.jl.
@@ -122,6 +122,8 @@ $DOC_LIST_OF_TESTS1
 - `:model_type`: Check `load_path` trait is correctly overloaded by attempting to
   re-import the type based on that trait's value.
 
+- `:traits`: Apply smoke tests to model trait values.
+
 $DOC_LIST_OF_TESTS2
 
 See also [`MLJTestInterface.make_binary`](@ref),
@@ -144,16 +146,18 @@ function test(model_types, data...; mod=Main, level=2, throw=false, verbosity=1,
         :name,
         :package_name,
         :model_type,
+        :traits,
         :model_instance,
         :fitted_machine,
         :operations,
-    ), NTuple{6, String}}}(undef, nmodels)
+    ), NTuple{7, String}}}(undef, nmodels)
 
     # summary table row corresponding to all tests skipped:
     row0 = (
         ; name="undefined",
         package_name= "undefined",
         model_type = "-",
+        traits = "-",
         model_instance = "-",
         fitted_machine = "-",
         operations = "-",
@@ -190,6 +194,15 @@ function test(model_types, data...; mod=Main, level=2, throw=false, verbosity=1,
         row = update!(summary, failures, row, i, :model_type, model_type, outcome)
         outcome == "×" && continue
 
+        # [traits]:
+        traits, outcome = MLJTestInterface.traits(
+            model_type;
+            throw,
+            verbosity,
+        )
+        row = update!(summary, failures, row, i, :traits, traits, outcome)
+        outcome == "×" && continue
+
         level > 1 || continue
 
         # [model_instance]:

src/test.jl

@@ -7,6 +7,7 @@ expected_report1 = (
     name = "ConstantClassifier",
     package_name = "MLJModels",
     model_type = "✓",
+    traits = "✓",
     model_instance = "✓",
     fitted_machine = "✓",
     operations = "predict",
@@ -16,6 +17,7 @@ expected_report2 = (
     name = "DeterministicConstantClassifier",
     package_name = "MLJModels",
     model_type = "✓",
+    traits = "✓",
     model_instance = "✓",
     fitted_machine = "✓",
     operations = "predict",
@@ -68,6 +70,7 @@ end
         name = "ConstantClassifier",
         package_name = "MLJModels",
         model_type = "✓",
+        traits = "✓",
         model_instance = "✓",
         fitted_machine = "×",
         operations = "-",
@@ -77,6 +80,7 @@ end
         name = "DeterministicConstantClassifier",
         package_name = "MLJModels",
         model_type = "✓",
+        traits = "✓",
         model_instance = "✓",
         fitted_machine = "✓",
         operations = "predict",
@@ -117,6 +121,7 @@ X, y = MLJTestInterface.make_binary()
     @test_logs(
         (:info, r"Testing ConstantClassifier"),
         (:info, r"model_type"),
+        (:info, r"traits"),
         (:info, r"model_instance"),
         (:info, r"fitted_machine"),
         (:info, r"operations"),
@@ -145,6 +150,7 @@ end
         name = "ConstantClassifier",
         package_name = "MLJModels",
         model_type = "✓",
+        traits = "✓",
         model_instance = "-",
         fitted_machine = "-",
         operations = "-",
@@ -164,6 +170,7 @@ end
         name = "ConstantClassifier",
         package_name = "MLJModels",
         model_type = "✓",
+        traits = "✓",
         model_instance = "✓",
         fitted_machine = "✓",
         operations = "predict",