refactor, using the new MLJTestInterface.jl as foundation

Author: ablaom

Project.toml

@@ -5,6 +5,7 @@ version = "0.2.6"
 
 [deps]
 MLJ = "add582a8-e3ab-11e8-2d5e-e98b27df1bc7"
+MLJTestInterface = "72560011-54dd-4dc2-94f3-c5de45b75ecd"
 NearestNeighborModels = "636a865e-7cf4-491e-846c-de09b730eb36"
 Pkg = "44cfe95a-1eb2-52ea-b672-e2afdf69b78f"
 Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"

src/MLJTestIntegration.jl

@@ -7,6 +7,9 @@ using Pkg
 using .Threads
 using Test
 using NearestNeighborModels
+import MLJTestInterface
+const MTI = MLJTestInterface
+import MLJTestInterface.attempt
 
 include("attemptors.jl")
 include("test.jl")

src/attemptors.jl

@@ -1,128 +1,15 @@
 const ERR_INCONSISTENT_RESULTS =
     "Different computational resources are giving different results. "
 
-"""
-    attempt(f, message; throw=false)
 
-Return `(f(), "✓") if `f()` executes without throwing an
-exception. Otherwise, return `(ex, "×"), where `ex` is the exception
-caught. Only truly throw the exception if `throw=true`.
-
-If `message` is not empty, then it is logged to `Info`, together with
-the second return value ("✓" or "×").
-
-
-"""
-function attempt(f, message; throw=false)
-    ret = try
-        (f(), "✓")
-    catch ex
-        throw && Base.throw(ex)
-        (ex, "×")
-    end
-    isempty(message) || @info message*last(ret)
-    return ret
-end
-
-finalize(message, verbosity) = verbosity < 2 ? "" : message
-
-
-# # ATTEMPTORS
-
-# TODO: Instead, in ****** below, use `MLJ.load_path`, after MLJModels
-# is updated to 0.16. And delete the two methods immediately
-# following. What's required will already be in MLJModels 0.15.10, but
-# the current implementation avoids an explicit MLJModels dependency
-# for MLJTestIntegration.
-load_path(model_type) = MLJ.load_path(model_type)
-function load_path(proxy::NamedTuple)
-    handle = (name=proxy.name, pkg=proxy.package_name)
-    return MLJ.MLJModels.INFO_GIVEN_HANDLE[handle][:load_path]
-end
+# # NEW ATTEMPTORS
 
 root(load_path) = split(load_path, '.') |> first
 
-function model_type(proxy, mod; throw=false, verbosity=1)
-    # check interface package really is in current environment:
-    message = "[:model_type] Loading model type "
-    model_type, outcome = attempt(finalize(message, verbosity); throw) do
-        load_path = MLJTestIntegration.load_path(proxy) # MLJ.load_path(proxy) *****
-        load_path_ex = load_path |> Meta.parse
-        api_pkg_ex = root(load_path) |> Symbol
-        import_ex = :(import $api_pkg_ex)
-        quote
-            $import_ex
-            $load_path_ex
-        end |>  mod.eval
-    end
-
-    # catch case of interface package not in current environment:
-    if outcome == "×" && model_type isa ArgumentError
-        # try to get the name of interface package; if this fails we
-        # catch the exception thrown but take no further
-        # action. Otherwise, we test if the original exception caught
-        # above, `model_type`, was triggered because of API package is
-        # missing from in environment.
-        api_pkg = try
-            load_path = MLJTestIntegration.load_path(proxy) # MLJ.load_path(proxy) *****
-            api_pkg = root(load_path)
-        catch
-            nothing
-        end
-        if !isnothing(api_pkg) &&
-               api_pkg != "unknown" &&
-               contains(model_type.msg, "$api_pkg not found in")
-            Base.throw(model_type)
-        end
-    end
-
-    return model_type, outcome
-end
-
-function model_instance(model_type; throw=false, verbosity=1)
-    message = "[:model_instance] Instantiating default model "
-    attempt(finalize(message, verbosity); throw)  do
-        model_type()
-    end
-end
-
-function fitted_machine(model, data...; throw=false, verbosity=1)
-    message = "[:fitted_machine] Fitting machine "
-    attempt(finalize(message, verbosity); throw)  do
-        mach = model isa Static ? machine(model) :
-                                  machine(model, data...)
-        fit!(mach, verbosity=-1)
-        MLJ.report(mach)
-        MLJ.fitted_params(mach)
-        mach
-    end
-end
-
-function operations(fitted_machine, data...; throw=false, verbosity=1)
-    message = "[:operations] Calling `predict`, `transform` and/or `inverse_transform` "
-    attempt(finalize(message, verbosity); throw)  do
-        operations = String[]
-        methods = MLJ.implemented_methods(fitted_machine.model)
-        if :predict in methods
-            predict(fitted_machine, first(data))
-            push!(operations, "predict")
-        end
-        if :transform in methods
-            W = transform(fitted_machine, first(data))
-            push!(operations, "transform")
-            if :inverse_transform in methods
-                inverse_transform(fitted_machine, W)
-                push!(operations, "inverse_transform")
-            end
-        end
-        join(operations, ", ")
-    end
-end
-
 function threshold_prediction(model, data...; throw=false, verbosity=1)
     message = "[:threshold_predictor] Calling fit!/predict for threshold predictor "*
         "test) "
-    attempt(finalize(message, verbosity); throw) do
+    attempt(MTI.finalize(message, verbosity); throw) do
         tmodel = BinaryThresholdPredictor(model)
         mach = machine(tmodel, data...)
         fit!(mach, verbosity=0)
@@ -134,7 +21,7 @@ function evaluation(measure, model, resources, data...; throw=false, verbosity=1
     L = length(resources)
     message = L > 1 ? "[:accelerated_evaluation] " : "[evaluation] "
     message *=  "Evaluating model performance using $L different resources. "
-    attempt(finalize(message, verbosity); throw) do
+    attempt(MTI.finalize(message, verbosity); throw) do
         es = map(resources) do resource
             evaluate(model, data...;
                      measure=measure,
@@ -157,7 +44,7 @@ function tuned_pipe_evaluation(
     verbosity=1,
 )
     message = "[:tuned_pipe_evaluation] Evaluating perfomance in a tuned pipeline "
-    attempt(finalize(message, verbosity); throw) do
+    attempt(MTI.finalize(message, verbosity); throw) do
         pipe = identity |> model
         tuned_pipe = TunedModel(
             models=fill(pipe, 3),
@@ -172,7 +59,7 @@ function tuned_pipe_evaluation(
 end
 
 function ensemble_prediction(model, data...; throw=false, verbosity=1)
-    attempt(finalize("[:ensemble_prediction] Ensembling ", verbosity); throw) do
+    attempt(MTI.finalize("[:ensemble_prediction] Ensembling ", verbosity); throw) do
         imodel = EnsembleModel(
             model=model,
             n=2,
@@ -186,7 +73,7 @@ end
 # the `model` must support iteration (`!isnothing(iteration_paramater(model))`)
 function iteration_prediction(measure, model, data...; throw=false, verbosity=1)
     message =  "[:iteration_prediction] Iterating with controls "
-    attempt(finalize(message, verbosity); throw) do
+    attempt(MTI.finalize(message, verbosity); throw) do
         imodel = IteratedModel(model=model,
                                measure=measure,
                                controls=[Step(1),
@@ -240,7 +127,7 @@ function stack_evaluation(
     isregressor = AbstractVector{Continuous} <: target_scitype
     measure = isregressor ? LPLoss(2) : BrierScore()
 
-    attempt(finalize(message, verbosity); throw) do
+    attempt(MTI.finalize(message, verbosity); throw) do
         es = map(resources) do resource
             stack = _stack(model, resource, isregressor)
             evaluate(

src/test.jl

@@ -253,27 +253,27 @@ function test(model_proxies, data...; mod=Main, level=2, throw=false, verbosity=
         row = merge(row0, (; name, package_name))
 
         # [model_type]:
-        model_type, outcome = MLJTestIntegration.model_type(model_proxy, mod; throw, verbosity)
+        model_type, outcome = MTI.model_type(model_proxy, mod; throw, verbosity)
         row = update(row, i, :model_type, model_type, outcome)
         outcome == "×" && continue
 
         level > 1 || continue
 
         # [model_instance]:
         model_instance, outcome =
-            MLJTestIntegration.model_instance(model_type; throw, verbosity)
+            MTI.model_instance(model_type; throw, verbosity)
         row = update(row, i, :model_instance, model_instance, outcome)
         outcome == "×" && continue
 
         # [fitted_machine]:
         fitted_machine, outcome =
-            MLJTestIntegration.fitted_machine(model_instance, data...; throw, verbosity)
+            MTI.fitted_machine(model_instance, data...; throw, verbosity)
         row = update(row, i, :fitted_machine, fitted_machine, outcome)
         outcome == "×" && continue
 
         # [operations]:
         operations, outcome =
-            MLJTestIntegration.operations(fitted_machine, data...; throw, verbosity)
+            MTI.operations(fitted_machine, data...; throw, verbosity)
         # special treatment to get list of operations in `summary`:
         if outcome == "×"
             row = update(row, i, :operations, operations, outcome)

test/attemptors.jl

@@ -1,30 +1,12 @@
-@testset "attempt()" begin
-    e = ArgumentError("elephant")
-    bad() = throw(e)
-    good() = 42
-
-    @test (@test_logs MLJTestIntegration.attempt(bad, "")) == (e, "×")
-    @test(@test_logs(
-        (:info, "look ×"),
-        MLJTestIntegration.attempt(bad, "look "),
-    )  == (e, "×"))
-    @test (@test_logs MLJTestIntegration.attempt(good, "")) == (42, "✓")
-    @test (@test_logs(
-        (:info, "look ✓"),
-        MLJTestIntegration.attempt(good, "look "),
-    )  == (42, "✓"))
-    @test_throws e MLJTestIntegration.attempt(bad, ""; throw=true)
-end
-
-@testset "model_type" begin
+@testset "model_type with model proxy instead of type" begin
 
     # test error thrown (not caught) if pkg missing from environment:
-    @test_throws ArgumentError MLJTestIntegration.model_type(
+    @test_throws ArgumentError MLJTestIntegration.MLJTestInterface.model_type(
         (name="PCA", package_name="MultivariateStats"),
         @__MODULE__
     )
 
-    M, outcome = MLJTestIntegration.model_type(
+    M, outcome = MLJTestIntegration.MLJTestInterface.model_type(
         (name="DecisionTreeClassifier", package_name="DecisionTree"),
         @__MODULE__;
         verbosity=0