add stack_evaluation; needs JuliaAI/MLJBase.jl#767

add stack_evaluation; needs JuliaAI/MLJBase.jl#767

rm target_scitype arg from stack_evaluation

put stack test into test()

oops

fix some bugs

separate out :accelerated_stack_evaluation test

more tweaks

oops

Author: ablaom

Project.toml

@@ -6,6 +6,7 @@ version = "0.1.0"
 [deps]
 MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"
 MLJTuning = "03970b2e-30c4-11ea-3135-d1576263f10f"
+NearestNeighborModels = "636a865e-7cf4-491e-846c-de09b730eb36"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
 

examples/bigtest/Project.toml

@@ -4,6 +4,7 @@ DataFrames = "a93c6f00-e57d-5684-b7b6-d8193f3e46c0"
 EvoTrees = "f6006082-12f8-11e9-0c9c-0d5d367ab1e5"
 InteractiveUtils = "b77e0a4c-d291-57a0-90e8-8db25a27a240"
 LightGBM = "7acf609c-83a4-11e9-1ffb-b912bcd3b04a"
+MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"
 MLJBase = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
 MLJClusteringInterface = "d354fa79-ed1c-40d4-88ef-b8c7bd1568af"
 MLJDecisionTreeInterface = "c6f25543-311c-4c74-83dc-3ea6d1015661"

examples/bigtest/notebook.jl

@@ -16,6 +16,7 @@ using DataFrames # for displaying tables
 # # Regression
 
 known_problems = models() do model
+    !model.is_pure_julia ||
     any([
         # https://github.com/lalvim/PartialLeastSquaresRegressor.jl/issues/29
         model.package_name == "PartialLeastSquaresRegressor",
@@ -28,14 +29,15 @@ end
 MLJTestIntegration.test_single_target_regressors(
     known_problems,
     ignore=true,
-    level=1
+    level=1,
 )
 
 fails1, report1 =
     MLJTestIntegration.test_single_target_regressors(
         known_problems,
         ignore=true,
-        level=4
+        level=4,
+        verbosity=2,
     )
 
 fails1 |> DataFrame
@@ -47,31 +49,42 @@ report1 |> DataFrame
 
 # # Classification
 
-# https://github.com/alan-turing-institute/MLJ.jl/issues/939
-known_problems = [
-    (name = "KernelPerceptronClassifier", package_name="BetaML"),
-    (name = "DecisionTreeClassifier", package_name="BetaML"),
-    (name = "PerceptronClassifier", package_name="BetaML"),
-    (name = "NuSVC", package_name="LIBSVM"),
-    (name="PegasosClassifier", package_name="BetaML"),
-    (name="RandomForestClassifier", package_name="BetaML"),
-    (name="SVMNuClassifier", package_name="ScikitLearn"),
-    (name="KernelPerceptronClassifier", package_name="BetaML"),
-    (name="LinearSVC", package_name="LIBSVM"),
-    (name= "MultinomialClassifier", "MLJLinearModels"),
-    (name="SVMLinearClassifier", package_name="ScikitLearn"),
-]
+known_problems = models() do model
+    !model.is_pure_julia ||
+    (name = model.name, package_name = model.package_name) in
+    [
+        # https://github.com/JuliaAI/MLJMultivariateStatsInterface.jl/issues/41
+        (name = "LDA", package_name = "MultivariateStats"),
+        (name = "SubspaceLDA", package_name = "MultivariateStats"),
+        (name = "BayesianLDA", package_name = "MultivariateStats"),
+        (name = "BayesianSubspaceLDA", package_name = "MultivariateStats"),
+
+        # https://github.com/alan-turing-institute/MLJ.jl/issues/939
+        (name = "DecisionTreeClassifier", package_name="BetaML"),
+        (name = "PerceptronClassifier", package_name="BetaML"),
+        (name = "NuSVC", package_name="LIBSVM"),
+        (name="PegasosClassifier", package_name="BetaML"),
+        (name="RandomForestClassifier", package_name="BetaML"),
+        (name="SVMNuClassifier", package_name="ScikitLearn"),
+        (name="KernelPerceptronClassifier", package_name="BetaML"),
+        (name="LinearSVC", package_name="LIBSVM"),
+        (name= "MultinomialClassifier", package_name="MLJLinearModels"),
+        (name="SVMLinearClassifier", package_name="ScikitLearn"),
+    ]
+end
 
 MLJTestIntegration.test_single_target_classifiers(
     known_problems,
     level=1,
     ignore=true,
 )
+
 fails2, report2 =
     MLJTestIntegration.test_single_target_classifiers(
         known_problems,
         ignore=true,
         level=4,
+        verbosity=2
     )
 
 fails2 |> DataFrame

src/MLJTestIntegration.jl

@@ -1,11 +1,12 @@
 module MLJTestIntegration
 
-const N_MODELS_FOR_REPEATABILITY_TEST = 3
+const N_MODELS_FOR_REPEATABILITY_TEST = 50
 
 using MLJ
 using Pkg
 using .Threads
 using Test
+using NearestNeighborModels
 
 include("attemptors.jl")
 include("test.jl")
@@ -14,7 +15,6 @@ include("dummy_model.jl")
 
 function __init__()
     global RESOURCES = (CPU1(), CPUThreads())
-    @info "Testing with $(nthreads()) threads. "
 end
 
 using .DummyModel

src/attemptors.jl

@@ -127,7 +127,9 @@ function threshold_prediction(model, data...; throw=false, verbosity=1)
 end
 
 function evaluation(measure, model, resources, data...; throw=false, verbosity=1)
-    message = "[:evaluation] Evaluating performance "
+    L = length(resources)
+    message = L > 1 ? "[:accelerated_evaluation] " : "[evaluation] "
+    message *=  "Evaluating model performance using with $L resources. "
     attempt(finalize(message, verbosity); throw) do
         es = map(resources) do accel
             evaluate(model, data...;
@@ -136,7 +138,7 @@ function evaluation(measure, model, resources, data...; throw=false, verbosity=1
                      acceleration=accel,
                      verbosity=0)
         end
-        ms = map(e->e.measurement, es)
+        ms = map(e->sort(e.per_fold[1]), es)
         m = first(ms)
         @assert all(≈(m), collect(ms)[2:end]) ERR_INCONSISTENT_RESULTS
         return first(es)
@@ -177,6 +179,7 @@ function ensemble_prediction(model, data...; throw=false, verbosity=1)
     end
 end
 
+# the `model` must support iteration (`!isnothing(iteration_paramater(model))`)
 function iteration_prediction(measure, model, data...; throw=false, verbosity=1)
     message =  "[:iteration_prediction] Iterating with controls "
     attempt(finalize(message, verbosity); throw) do
@@ -190,3 +193,59 @@ function iteration_prediction(measure, model, data...; throw=false, verbosity=1)
         predict(mach, first(data))
     end
 end
+
+# the `model` can only be single-target deterministic regressor or
+# probabilistic classifier.
+function stack_evaluation(
+    model,
+    resources,
+    data...;
+    throw=false,
+    verbosity=1
+)
+    L = length(resources)
+    message = L > 1 ? "[:accelerated_stack_evaluation] " : "[stack_evaluation] "
+    message *=  "Evaluating a stack containing model "*
+        "with $L resources. "
+    target_scitype = MLJ.target_scitype(model)
+    if  AbstractVector{Continuous} <: target_scitype
+        models = (knn1=KNNRegressor(K=4),
+                  knn2=KNNRegressor(K=6),
+                  model=model)
+        metalearner = KNNRegressor()
+        measure = LPLoss(2)
+    else
+        models = (knn1=KNNClassifier(K=4),
+                  knn2=KNNClassifier(K=6),
+                  model=model)
+        metalearner = KNNClassifier()
+        measure = BrierScore()
+        # models = (tree=DecisionTreeClassifier(),
+        #           knn=KNNClassifier(K=6),
+        #           model=model)
+        # metalearner = KNNClassifier()
+        # measure = BrierScore()
+    end
+    attempt(finalize(message, verbosity); throw) do
+        es = map(resources) do accel
+            mystack = Stack(
+                ; metalearner,
+                resampling=CV(;nfolds=3),
+                acceleration=accel,
+                models...)
+
+            evaluate(
+                mystack,
+                data...;
+                measure=measure,
+                resampling=Holdout(),
+                verbosity=0,
+            )
+        end |> collect
+        ms = map(e->sort(e.per_fold[1]), es)
+        m = first(ms)
+#        @show ms
+        @assert all(≈(m), ms[2:end]) ERR_INCONSISTENT_RESULTS
+        first(es)
+    end
+end

src/special_cases.jl

@@ -18,7 +18,7 @@ strip(proxy) = (name=proxy.name, package_name=proxy.package_name)
 function actual_proxies(raw_proxies, data, ignore, verbosity)
     if !(raw_proxies isa Vector)
         raw_proxies = [raw_proxies, ]
-    end 
+    end
     proxies = strip.(raw_proxies)
     from_registry = strip.(models(matching(data...)))
     if ignore
@@ -34,7 +34,7 @@ function actual_proxies(raw_proxies, data, ignore, verbosity)
 end
 
 function _test(proxies, data; ignore::Bool=false, verbosity=1, kwargs...)
-    test(actual_proxies(proxies, data, ignore, verbosity), data...; kwargs...)
+    test(actual_proxies(proxies, data, ignore, verbosity), data...; verbosity, kwargs...)
 end
 _test(data; ignore=true, kwargs...) = _test([], data; ignore, kwargs...)