change signature of convenience functions, with ignore now Bool

Author: ablaom

examples/bigtest/notebook.jl

@@ -15,7 +15,7 @@ using DataFrames # for displaying tables
 
 # # Regression
 
-known_issues = models() do model
+known_problems = models() do model
     any([
         # https://github.com/lalvim/PartialLeastSquaresRegressor.jl/issues/29
         model.package_name == "PartialLeastSquaresRegressor",
@@ -25,25 +25,41 @@ known_issues = models() do model
     ])
 end
 
-MLJTestIntegration.test_single_target_regressors(ignore=known_issues, level=1)
-fails, summary =
-    MLJTestIntegration.test_single_target_regressors(ignore=known_issues, level=3)
+MLJTestIntegration.test_single_target_regressors(
+    known_problems,
+    ignore=true,
+    level=1
+)
+
+fails, report =
+    MLJTestIntegration.test_single_target_regressors(
+        known_problems,
+        ignore=true,
+        level=3
+    )
 
 @test isempty(fails)
-summary |> DataFrame
+report |> DataFrame
 
 
 # # Classification
 
 # https://github.com/alan-turing-institute/MLJ.jl/issues/939
-known_issues = [
+known_problems = [
     (name = "DecisionTreeClassifier", package_name="BetaML"),
+    (name = "PerceptronClassifier", package_name="BetaML"),
     (name = "NuSVC", package_name="LIBSVM"),
     (name="PegasosClassifier", package_name="BetaML"),
     (name="RandomForestClassifier", package_name="BetaML"),
     (name="SVMNuClassifier", package_name="ScikitLearn"),
 ]
 
-MLJTestIntegration.test_single_target_classifiers(ignore=known_issues, level=1)
-fails, summary =
-    MLJTestIntegration.test_single_target_classifiers(ignore=known_issues, level=3)
+MLJTestIntegration.test_single_target_classifiers(
+    known_problems,
+    level=1
+)
+fails, report =
+    MLJTestIntegration.test_single_target_classifiers(
+        known_problems,
+        level=3,
+    )

src/special_cases.jl

@@ -1,25 +1,31 @@
 # # HELPERS
 
-_strip(proxy) = (name=proxy.name, package_name=proxy.package_name)
-
-function _filter(proxies, bad)
-    sbad = _strip.(bad)
-    filter(proxies) do proxy
-        !(_strip(proxy) in sbad)
-    end
+function warn_not_testing_these(models)
+    "Not testing the following models, as incompatible with testing data:\n"*
+        "$models"
 end
 
-# fallback:
-function _test(data, ignore; kwargs...)
-    proxies = _filter(models(matching(data...)), ignore)
-    test(proxies, data...; kwargs...)
+strip(proxy) = (name=proxy.name, package_name=proxy.package_name)
+
+function actual_proxies(raw_proxies, data, ignore, verbosity)
+    proxies = strip.(raw_proxies)
+    from_registry = strip.(models(matching(data...)))
+    if ignore
+        actual_proxies = setdiff(from_registry, proxies)
+    else
+        actual_proxies = intersect(proxies, from_registry)
+        rejected = setdiff(proxies, actual_proxies)
+        if !isempty(rejected) && verbosity > 0
+            @warn warn_not_testing_these(rejected)
+        end
+    end
+    return actual_proxies
 end
 
-# when there are no models to exclude:
-function _test(data, ignore::Nothing; kwargs...)
-    proxies = models(matching(data...))
-    test(proxies, data...; kwargs...)
+function _test(proxies, data; ignore::Bool=false, verbosity=1, kwargs...)
+    test(actual_proxies(proxies, data, ignore, verbosity), data...; kwargs...)
 end
+_test(data; ignore=true, kwargs...) = _test([], data; ignore, kwargs...)
 
 
 # # SINGLE TARGET CLASSIFICATION
@@ -31,8 +37,8 @@ function _make_binary()
     return X, y
 end
 
-test_single_target_classifiers(; ignore=nothing, kwargs...) =
-    _test(_make_binary(), ignore; kwargs...)
+test_single_target_classifiers(args...; kwargs...) =
+    _test(args..., _make_binary(); kwargs...)
 
 
 # # SINGLE TARGET REGRESSION
@@ -43,8 +49,8 @@ function _make_baby_boston()
     return X, y
 end
 
-test_single_target_regressors(; ignore=nothing, kwargs...) =
-    _test(_make_baby_boston(), ignore; kwargs...)
+test_single_target_regressors(args...; kwargs...) =
+    _test(args..., _make_baby_boston(); kwargs...)
 
 
 # # SINGLE TARGET COUNT REGRESSORS
@@ -55,13 +61,13 @@ function _make_count()
     return X, y
 end
 
-test_single_target_count_regressors(; ignore=nothing, kwargs...) =
-    _test(_make_count(), ignore; kwargs...)
+test_single_target_count_regressors(args...; kwargs...) =
+    _test(args..., _make_count(); kwargs...)
 
 
 # # CONTINUOUS TABLE TRANSFORMERS
 
 _make_transformer() = (first(_make_baby_boston()),)
 
-test_continuous_table_transformers(; ingore=nothing, kwargs...) =
-    _test(_make_transformer(), ignore; kwargs...)
+test_continuous_table_transformers(args...; kwargs...) =
+    _test(args..., _make_transformer(); kwargs...)

test/runtests.jl

@@ -3,10 +3,12 @@ using Pkg
 using MLJTestIntegration
 using MLJTestIntegration.MLJ
 using MLJTestIntegration.MLJ.MLJModels
+const MTI = MLJTestIntegration
 
 # enable conditional testing of modules by providing test_args
 # e.g. `Pkg.test("MLJBase", test_args=["misc"])`
-RUN_ALL_TESTS = isempty(ARGS)
+
+const RUN_ALL_TESTS = isempty(ARGS)
 macro conditional_testset(name, expr)
     name = string(name)
     esc(quote

test/special_cases.jl

@@ -1,18 +1,26 @@
-@testset "_filter" begin
-    proxies = [
-        (name="1", package_name="A", extra="cat"),
-        (name="1", package_name="B", extra="mouse"),
-        (name="2", package_name="B", extra="dog"),
-        (name="1", package_name="C", extra="rat"),
-    ]
+classifiers = [
+    (name = "ConstantClassifier", package_name = "MLJModels"),
+    (name = "DeterministicConstantClassifier", package_name = "MLJModels")
+]
 
-    bad = [
-        (name="1", package_name="A"),
-        (name="1", package_name="B"),
-    ]
+regressors = [
+    (name = "ConstantRegressor", package_name = "MLJModels"),
+    (name = "DeterministicConstantRegressor", package_name = "MLJModels")
+]
 
-    @test MLJTestIntegration._filter(proxies, bad) == [
-        (name="2", package_name="B", extra="dog"),
-        (name="1", package_name="C", extra="rat"),
-    ]
+@testset "actual_proxies" begin
+    data =  MTI._make_baby_boston()
+    proxies = @test_logs MTI.actual_proxies(regressors, data, false, 1)
+    @test proxies == regressors
+    proxies2 = @test_logs MTI.actual_proxies(regressors, data, true, 1)
+    @test proxies2 == setdiff(MTI.strip.(models(matching(data...))), regressors)
+    proxies = @test_logs(
+        (:warn, MTI.warn_not_testing_these(classifiers)),
+        MTI.actual_proxies(vcat(regressors, classifiers), data, false, 1),
+    )
+    @test proxies == regressors
+    proxies = @test_logs(
+        MTI.actual_proxies(vcat(regressors, classifiers), data, true, 1),
+    )
+    @test proxies == proxies2
 end

test/test.jl

@@ -1,4 +1,3 @@
-# grab some classifiers from MLJModels:
 classifiers = [
     (name = "ConstantClassifier", package_name = "MLJModels"),
     (name = "DeterministicConstantClassifier", package_name = "MLJModels")