Merge pull request #42 from JuliaAI/verbosity-bug

Fix verbosity bug

Author: ablaom

.github/workflows/ci_nightly.yml

@@ -1,7 +1,7 @@
 name = "MLJTestIntegration"
 uuid = "697918b4-fdc1-4f9e-8ff9-929724cee270"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "0.5.0"
+version = "0.5.1"
 
 [deps]
 MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"

Project.toml

@@ -491,12 +491,10 @@ function test(model; mod=Main, level=2, throw=false, verbosity=1,)
             println()
         end
         if !isempty(fails)
-            if verbosity > 0
-                @warn "\n Failures encounted for these tests:"
-                for e in fails
-                    @warn "\t [$(e.test)]"
-                    push!(exceptions, e)
-                end
+            verbosity > 0 && @warn "\n Failures encounted for these tests:"
+            for e in fails
+                @warn "\t [$(e.test)]"
+                push!(exceptions, e)
             end
         end
     end

src/test.jl

@@ -4,6 +4,7 @@ using MLJTestIntegration
 using MLJ
 using MLJModels
 const MTI = MLJTestIntegration
+const MMI = MLJ.MLJBase.MLJModelInterface
 
 classifiers = [
     (name = "ConstantClassifier", package_name = "MLJModels"),
@@ -315,4 +316,18 @@ end
     )
 end
 
+struct JunkModel <: Deterministic end
+
+MMI.fit(::JunkModel, verbosity, X, y) = error("I am always broken")
+MMI.input_scitype(::Type{JunkModel}) = Table(Continuous)
+MMI.target_scitype(::Type{JunkModel}) = AbstractVector{<:Continuous}
+
+@testset "test(model, ...) catches failure" begin
+    @test(@test_logs(
+        (:warn, MTI.WARN_FAILURES_ENCOUNTERED),
+        match_mode=:any,
+        !isempty(MTI.test(JunkModel; mod=@__MODULE__, verbosity=0, throw=false)),
+    ))
+end
+
 true