primary iters

Author: be-marc

tests/testthat/test_Measure.R

@@ -275,38 +275,29 @@ test_that("measure weights", {
 
 })
 
-
-
-
 test_that("primary iters are respected", {
   task = tsk("sonar")
-  resampling = rsmp("cv")$instantiate(task)
-  train_sets = map(1:10, function(i) resampling$train_set(i))
-  test_sets = map(1:10, function(i) resampling$train_set(i))
-  r1 = rsmp("custom")$instantiate(task, train_sets = train_sets, test_sets = test_sets)
-  get_private(r1, ".primary_iters") = 1:2
-  r2 = rsmp("custom")$instantiate(task, train_sets = train_sets[1:2], test_sets = test_sets[1:2])
-  r3 = rsmp("custom")$instantiate(task, train_sets = train_sets, test_sets = test_sets)
-
   learner = lrn("classif.rpart", predict_type = "prob")
+  resampling = rsmp("cv", folds = 10)
+  resampling$instantiate(task)
+  get_private(resampling, ".primary_iters") = 1:2
 
-  rr1 = resample(task, learner, r1, store_models = TRUE)
-  rr2 = resample(task, learner, r2, store_models = TRUE)
-  rr3 = resample(task, learner, r3, store_models = TRUE)
-
+  rr = resample(task, learner, resampling)
   m = msr("classif.acc")
   m$average = "macro"
-  expect_equal(rr1$aggregate(), rr2$aggregate())
-  m$average = "micro"
-  expect_equal(rr1$aggregate(), rr2$aggregate())
+  scores = rr$score(m)$classif.acc
+
+  # macro aggregation
+  expect_equal(unname(rr$aggregate(m)), mean(scores[1:2]))
+  expect_true(unname(rr$aggregate(m)) != mean(scores))
+
+  # micro aggregation
+  pred_micro = do.call(c, rr$predictions()[1:2])
+  scores = pred_micro$score(m)
+  expect_equal(unname(m$score(pred_micro)), unname(scores))
 
   jaccard = msr("sim.jaccard")
-  expect_error(rr1$aggregate(jaccard), "primary_iters")
-  expect_no_error(rr2$aggregate(jaccard))
-  jaccard$properties = c(jaccard$properties, "primary_iters")
-  x1 = rr1$aggregate(jaccard)
-  x2 = rr3$aggregate(jaccard)
-  expect_equal(x1, x2)
+  expect_error(rr$aggregate(jaccard), "primary_iters")
 })
 
 test_that("no predict_sets required (#1094)", {