Merge pull request #1010 from JuliaAI/resampling-option

Allow tuples in the `resampling` option for `evaluate`

Author: ablaom

Project.toml

@@ -1,7 +1,7 @@
 name = "MLJBase"
 uuid = "a7f614a8-145f-11e9-1d2a-a57a1082229d"
 authors = ["Anthony D. Blaom <[email protected]>"]
-version = "1.8.2"
+version = "1.9.0"
 
 [deps]
 CategoricalArrays = "324d7699-5711-5eae-9e2f-1d82baa6b597"

src/resampling.jl

@@ -1,8 +1,11 @@
-# TYPE ALIASES
+# # LOCAL TYPE ALIASES
 
 const AbstractRow = Union{AbstractVector{<:Integer}, Colon}
-const TrainTestPair = Tuple{AbstractRow,AbstractRow}
-const TrainTestPairs = AbstractVector{<:TrainTestPair}
+const TrainTestPair = Tuple{AbstractRow, AbstractRow}
+const TrainTestPairs = Union{
+    Tuple{Vararg{TrainTestPair}},
+    AbstractVector{<:TrainTestPair},
+}
 
 
 # # ERROR MESSAGES
@@ -93,6 +96,19 @@ const ERR_NEED_TARGET = ArgumentError(
     """
 )
 
+const ERR_BAD_RESAMPLING_OPTION = ArgumentError(
+    "`resampling` must be an "*
+        "`MLJ.ResamplingStrategy` or a vector (or tuple) of tuples "*
+        "of the form `(train_rows, test_rows)`"
+)
+
+const ERR_EMPTY_RESAMPLING_OPTION = ArgumentError(
+    "`resampling` cannot be emtpy. It must be an "*
+        "`MLJ.ResamplingStrategy` or a vector (or tuple) of tuples "*
+        "of the form `(train_rows, test_rows)`"
+)
+
+
 # ==================================================================
 ## RESAMPLING STRATEGIES
 
@@ -1402,10 +1418,6 @@ end
 # ------------------------------------------------------------
 # Core `evaluation` method, operating on train-test pairs
 
-const AbstractRow = Union{AbstractVector{<:Integer}, Colon}
-const TrainTestPair = Tuple{AbstractRow, AbstractRow}
-const TrainTestPairs = AbstractVector{<:TrainTestPair}
-
 _view(::Nothing, rows) = nothing
 _view(weights, rows) = view(weights, rows)
 
@@ -1434,11 +1446,8 @@ function evaluate!(
     # Note: `rows` and `repeats` are only passed to the final `PeformanceEvaluation`
     # object to be returned and are not otherwise used here.
 
-    if !(resampling isa TrainTestPairs)
-        error("`resampling` must be an "*
-              "`MLJ.ResamplingStrategy` or an vector tuples "*
-              "of the form `(train_rows, test_rows)`")
-    end
+    isempty(resampling) && throw(ERR_EMPTY_RESAMPLING_OPTION)
+    resampling isa TrainTestPairs || throw(ERR_BAD_RESAMPLING_OPTION)
 
     X = mach.args[1]()
     y = mach.args[2]()

test/resampling.jl

@@ -276,6 +276,23 @@ end
         model = DeterministicConstantRegressor()
         mach  = machine(model, X, y, cache=cache)
 
+        # check catch for bad `resampling` options:
+        @test_throws(
+            MLJBase.ERR_BAD_RESAMPLING_OPTION,
+            evaluate(model, X, y; resampling="junk", verbosity=0, acceleration=accel),
+        )
+        @test_throws(
+            MLJBase.ERR_EMPTY_RESAMPLING_OPTION,
+            evaluate(model, X, y; resampling=[], verbosity=0, acceleration=accel),
+        )
+
+        # check we can provide tuples of pairs, instead of vectors of pairs, in
+        # `resampling` option:
+        e1 = evaluate(model, X, y; resampling, verbosity=0, acceleration=accel)
+        e2 = evaluate(model, X, y;
+                      resampling=Tuple(resampling), verbosity=0, acceleration=accel)
+        @test e1.measurement[1] ≈ e2.measurement[1]
+
         # check detection of incompatible measure (cross_entropy):
         @test_throws(
             MLJBase.err_incompatible_prediction_types(model, cross_entropy),