diff --git a/DESCRIPTION b/DESCRIPTION index ef7b8ce..d3cf2db 100644 --- a/DESCRIPTION +++ b/DESCRIPTION @@ -20,7 +20,7 @@ URL: https://github.com/tidymodels/finetune, BugReports: https://github.com/tidymodels/finetune/issues Depends: R (>= 4.1), - tune (>= 1.2.0) + tune (>= 1.3.0.9006) Imports: cli, dials (>= 0.3.0), diff --git a/tests/testthat/_snaps/sa-misc.md b/tests/testthat/_snaps/sa-misc.md index 52df4bf..3dce56b 100644 --- a/tests/testthat/_snaps/sa-misc.md +++ b/tests/testthat/_snaps/sa-misc.md @@ -6,9 +6,9 @@ Message Optimizing roc_auc Initial best: 0.85731 - 1 ( ) accept suboptimal roc_auc=0.85682 (+/-0.01022) - 2 ( ) accept suboptimal roc_auc=0.85238 (+/-0.01078) - 3 ( ) accept suboptimal roc_auc=0.85138 (+/-0.0109) + 1 ( ) accept suboptimal roc_auc=0.85432 (+/-0.01039) + 2 <3 new best roc_auc=0.86674 (+/-0.009097) + 3 <3 new best roc_auc=0.87245 (+/-0.008363) --- @@ -18,9 +18,9 @@ Message Optimizing roc_auc Initial best: 0.85325 - 1 ( ) accept suboptimal roc_auc=0.85313 (+/-0.0106) - 2 ( ) accept suboptimal roc_auc=0.85181 (+/-0.01065) - 3 ( ) accept suboptimal roc_auc=0.85165 (+/-0.01055) + 1 ( ) accept suboptimal roc_auc=0.85268 (+/-0.01056) + 2 <3 new best roc_auc=0.85618 (+/-0.01015) + 3 <3 new best roc_auc=0.85799 (+/-0.009991) --- @@ -30,9 +30,9 @@ Message Optimizing roc_auc Initial best: 0.86616 - 1 ( ) accept suboptimal roc_auc=0.86399 (+/-0.01081) - 2 <3 new best roc_auc=0.86768 (+/-0.009563) - 3 <3 new best roc_auc=0.87329 (+/-0.008273) + 1 ( ) accept suboptimal roc_auc=0.86568 (+/-0.01027) + 2 <3 new best roc_auc=0.86654 (+/-0.009714) + 3 <3 new best roc_auc=0.8688 (+/-0.009487) --- @@ -42,7 +42,15 @@ Message Optimizing roc_auc Initial best: 0.86616 - 1 ( ) accept suboptimal roc_auc=0.86399 (+/-0.01081) - 2 <3 new best roc_auc=0.86768 (+/-0.009563) - 3 <3 new best roc_auc=0.87329 (+/-0.008273) + 1 ( ) accept suboptimal roc_auc=0.86568 (+/-0.01027) + 2 <3 new best roc_auc=0.86654 (+/-0.009714) + 3 <3 new best roc_auc=0.8688 (+/-0.009487) + +# tune_sim_anneal with wrong type + + Code + tune_sim_anneal(1) + Condition + Error in `tune_sim_anneal()`: + ! The first argument to `tune_sim_anneal()` should be either a model or workflow. diff --git a/tests/testthat/_snaps/sa-overall.md b/tests/testthat/_snaps/sa-overall.md index 4e1b77b..aead642 100644 --- a/tests/testthat/_snaps/sa-overall.md +++ b/tests/testthat/_snaps/sa-overall.md @@ -12,79 +12,43 @@ Optimizing roc_auc Initial best: 0.73008 i Fold1, Repeat1: preprocessor 1/1 - v Fold1, Repeat1: preprocessor 1/1 i Fold1, Repeat1: preprocessor 1/1, model 1/1 - v Fold1, Repeat1: preprocessor 1/1, model 1/1 - i Fold1, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat1: preprocessor 1/1 - v Fold2, Repeat1: preprocessor 1/1 i Fold2, Repeat1: preprocessor 1/1, model 1/1 - v Fold2, Repeat1: preprocessor 1/1, model 1/1 - i Fold2, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat1: preprocessor 1/1 - v Fold3, Repeat1: preprocessor 1/1 i Fold3, Repeat1: preprocessor 1/1, model 1/1 - v Fold3, Repeat1: preprocessor 1/1, model 1/1 - i Fold3, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold1, Repeat2: preprocessor 1/1 - v Fold1, Repeat2: preprocessor 1/1 i Fold1, Repeat2: preprocessor 1/1, model 1/1 - v Fold1, Repeat2: preprocessor 1/1, model 1/1 - i Fold1, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat2: preprocessor 1/1 - v Fold2, Repeat2: preprocessor 1/1 i Fold2, Repeat2: preprocessor 1/1, model 1/1 - v Fold2, Repeat2: preprocessor 1/1, model 1/1 - i Fold2, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat2: preprocessor 1/1 - v Fold3, Repeat2: preprocessor 1/1 i Fold3, Repeat2: preprocessor 1/1, model 1/1 - v Fold3, Repeat2: preprocessor 1/1, model 1/1 - i Fold3, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat2: preprocessor 1/1, model 1/1 (predictions) - 1 ( ) accept suboptimal roc_auc=0.72145 (+/-0.003605) + 1 <3 new best roc_auc=0.74036 (+/-0.002433) i Fold1, Repeat1: preprocessor 1/1 - v Fold1, Repeat1: preprocessor 1/1 i Fold1, Repeat1: preprocessor 1/1, model 1/1 - v Fold1, Repeat1: preprocessor 1/1, model 1/1 - i Fold1, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat1: preprocessor 1/1 - v Fold2, Repeat1: preprocessor 1/1 i Fold2, Repeat1: preprocessor 1/1, model 1/1 - v Fold2, Repeat1: preprocessor 1/1, model 1/1 - i Fold2, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat1: preprocessor 1/1 - v Fold3, Repeat1: preprocessor 1/1 i Fold3, Repeat1: preprocessor 1/1, model 1/1 - v Fold3, Repeat1: preprocessor 1/1, model 1/1 - i Fold3, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold1, Repeat2: preprocessor 1/1 - v Fold1, Repeat2: preprocessor 1/1 i Fold1, Repeat2: preprocessor 1/1, model 1/1 - v Fold1, Repeat2: preprocessor 1/1, model 1/1 - i Fold1, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat2: preprocessor 1/1 - v Fold2, Repeat2: preprocessor 1/1 i Fold2, Repeat2: preprocessor 1/1, model 1/1 - v Fold2, Repeat2: preprocessor 1/1, model 1/1 - i Fold2, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat2: preprocessor 1/1 - v Fold3, Repeat2: preprocessor 1/1 i Fold3, Repeat2: preprocessor 1/1, model 1/1 - v Fold3, Repeat2: preprocessor 1/1, model 1/1 - i Fold3, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat2: preprocessor 1/1, model 1/1 (predictions) - 2 <3 new best roc_auc=0.73173 (+/-0.003018) + 2 ( ) accept suboptimal roc_auc=0.73543 (+/-0.009624) # variable interface @@ -100,79 +64,43 @@ Optimizing roc_auc Initial best: 0.73008 i Fold1, Repeat1: preprocessor 1/1 - v Fold1, Repeat1: preprocessor 1/1 i Fold1, Repeat1: preprocessor 1/1, model 1/1 - v Fold1, Repeat1: preprocessor 1/1, model 1/1 - i Fold1, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat1: preprocessor 1/1 - v Fold2, Repeat1: preprocessor 1/1 i Fold2, Repeat1: preprocessor 1/1, model 1/1 - v Fold2, Repeat1: preprocessor 1/1, model 1/1 - i Fold2, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat1: preprocessor 1/1 - v Fold3, Repeat1: preprocessor 1/1 i Fold3, Repeat1: preprocessor 1/1, model 1/1 - v Fold3, Repeat1: preprocessor 1/1, model 1/1 - i Fold3, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold1, Repeat2: preprocessor 1/1 - v Fold1, Repeat2: preprocessor 1/1 i Fold1, Repeat2: preprocessor 1/1, model 1/1 - v Fold1, Repeat2: preprocessor 1/1, model 1/1 - i Fold1, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat2: preprocessor 1/1 - v Fold2, Repeat2: preprocessor 1/1 i Fold2, Repeat2: preprocessor 1/1, model 1/1 - v Fold2, Repeat2: preprocessor 1/1, model 1/1 - i Fold2, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat2: preprocessor 1/1 - v Fold3, Repeat2: preprocessor 1/1 i Fold3, Repeat2: preprocessor 1/1, model 1/1 - v Fold3, Repeat2: preprocessor 1/1, model 1/1 - i Fold3, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat2: preprocessor 1/1, model 1/1 (predictions) - 1 ( ) accept suboptimal roc_auc=0.72145 (+/-0.003605) + 1 <3 new best roc_auc=0.74036 (+/-0.002433) i Fold1, Repeat1: preprocessor 1/1 - v Fold1, Repeat1: preprocessor 1/1 i Fold1, Repeat1: preprocessor 1/1, model 1/1 - v Fold1, Repeat1: preprocessor 1/1, model 1/1 - i Fold1, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat1: preprocessor 1/1 - v Fold2, Repeat1: preprocessor 1/1 i Fold2, Repeat1: preprocessor 1/1, model 1/1 - v Fold2, Repeat1: preprocessor 1/1, model 1/1 - i Fold2, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat1: preprocessor 1/1 - v Fold3, Repeat1: preprocessor 1/1 i Fold3, Repeat1: preprocessor 1/1, model 1/1 - v Fold3, Repeat1: preprocessor 1/1, model 1/1 - i Fold3, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold1, Repeat2: preprocessor 1/1 - v Fold1, Repeat2: preprocessor 1/1 i Fold1, Repeat2: preprocessor 1/1, model 1/1 - v Fold1, Repeat2: preprocessor 1/1, model 1/1 - i Fold1, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat2: preprocessor 1/1 - v Fold2, Repeat2: preprocessor 1/1 i Fold2, Repeat2: preprocessor 1/1, model 1/1 - v Fold2, Repeat2: preprocessor 1/1, model 1/1 - i Fold2, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat2: preprocessor 1/1 - v Fold3, Repeat2: preprocessor 1/1 i Fold3, Repeat2: preprocessor 1/1, model 1/1 - v Fold3, Repeat2: preprocessor 1/1, model 1/1 - i Fold3, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat2: preprocessor 1/1, model 1/1 (predictions) - 2 <3 new best roc_auc=0.73173 (+/-0.003018) + 2 ( ) accept suboptimal roc_auc=0.73543 (+/-0.009624) --- @@ -183,9 +111,9 @@ Message There were 2 previous iterations Optimizing roc_auc - 2 v initial roc_auc=0.73173 (+/-0.003018) - 3 <3 new best roc_auc=0.74172 (+/-0.008775) - 4 <3 new best roc_auc=0.7909 (+/-0.009887) + 2 v initial roc_auc=0.74036 (+/-0.002433) + 3 <3 new best roc_auc=0.75103 (+/-0.009282) + 4 <3 new best roc_auc=0.76617 (+/-0.01037) --- @@ -205,19 +133,19 @@ set.seed(40) rf_res_finetune <- tune_sim_anneal(wf_rf, resamples = bt, initial = rf_res) Message - i Creating pre-processing data to finalize unknown parameter: mtry + i Creating pre-processing data to finalize 1 unknown parameter: "mtry" Optimizing roc_auc - Initial best: 0.84994 - 1 ( ) accept suboptimal roc_auc=0.84375 (+/-0.007727) - 2 + better suboptimal roc_auc=0.84943 (+/-0.007036) - 3 ( ) accept suboptimal roc_auc=0.84371 (+/-0.007903) - 4 + better suboptimal roc_auc=0.84825 (+/-0.008036) - 5 ( ) accept suboptimal roc_auc=0.84479 (+/-0.00814) - 6 + better suboptimal roc_auc=0.84816 (+/-0.007283) - 7 ( ) accept suboptimal roc_auc=0.84381 (+/-0.007999) - 8 <3 new best roc_auc=0.85014 (+/-0.007172) - 9 ( ) accept suboptimal roc_auc=0.84344 (+/-0.007818) - 10 + better suboptimal roc_auc=0.84802 (+/-0.007281) + Initial best: 0.84917 + 1 ( ) accept suboptimal roc_auc=0.84467 (+/-0.007843) + 2 <3 new best roc_auc=0.84956 (+/-0.007013) + 3 ( ) accept suboptimal roc_auc=0.84384 (+/-0.008042) + 4 + better suboptimal roc_auc=0.84952 (+/-0.00757) + 5 ( ) accept suboptimal roc_auc=0.84357 (+/-0.008148) + 6 + better suboptimal roc_auc=0.84848 (+/-0.007499) + 7 ( ) accept suboptimal roc_auc=0.84391 (+/-0.008097) + 8 + better suboptimal roc_auc=0.84853 (+/-0.007036) + 9 ( ) accept suboptimal roc_auc=0.84471 (+/-0.007812) + 10 x restart from best roc_auc=0.84935 (+/-0.007313) --- @@ -225,19 +153,19 @@ set.seed(40) rf_res_finetune <- tune_sim_anneal(wf_rf, resamples = bt) Message - i Creating pre-processing data to finalize unknown parameter: mtry + i Creating pre-processing data to finalize 1 unknown parameter: "mtry" Optimizing roc_auc - Initial best: 0.84418 - 1 <3 new best roc_auc=0.84839 (+/-0.007753) - 2 ( ) accept suboptimal roc_auc=0.84384 (+/-0.008085) - 3 <3 new best roc_auc=0.84857 (+/-0.007615) - 4 ( ) accept suboptimal roc_auc=0.8435 (+/-0.007746) - 5 + better suboptimal roc_auc=0.84804 (+/-0.00774) - 6 ( ) accept suboptimal roc_auc=0.84338 (+/-0.007515) - 7 <3 new best roc_auc=0.84923 (+/-0.007371) - 8 ( ) accept suboptimal roc_auc=0.84389 (+/-0.007938) - 9 <3 new best roc_auc=0.84926 (+/-0.007163) - 10 ( ) accept suboptimal roc_auc=0.84397 (+/-0.00741) + Initial best: 0.84369 + 1 <3 new best roc_auc=0.84929 (+/-0.007889) + 2 ( ) accept suboptimal roc_auc=0.84461 (+/-0.008084) + 3 + better suboptimal roc_auc=0.84785 (+/-0.007989) + 4 ( ) accept suboptimal roc_auc=0.84325 (+/-0.007817) + 5 + better suboptimal roc_auc=0.84917 (+/-0.007376) + 6 ( ) accept suboptimal roc_auc=0.84458 (+/-0.007566) + 7 + better suboptimal roc_auc=0.84871 (+/-0.007483) + 8 ( ) accept suboptimal roc_auc=0.84431 (+/-0.007694) + 9 x restart from best roc_auc=0.84861 (+/-0.007929) + 10 ( ) accept suboptimal roc_auc=0.84522 (+/-0.008113) # incompatible parameter objects diff --git a/tests/testthat/test-sa-misc.R b/tests/testthat/test-sa-misc.R index 85bb246..db90777 100644 --- a/tests/testthat/test-sa-misc.R +++ b/tests/testthat/test-sa-misc.R @@ -1,3 +1,5 @@ +skip_if_not_installed("tune", minimum_version = "1.3.0.9006") + ## ----------------------------------------------------------------------------- test_that("tune_sim_anneal interfaces", { diff --git a/tests/testthat/test-sa-overall.R b/tests/testthat/test-sa-overall.R index ed8f0a4..5f90a62 100644 --- a/tests/testthat/test-sa-overall.R +++ b/tests/testthat/test-sa-overall.R @@ -1,3 +1,5 @@ +skip_if_not_installed("tune", minimum_version = "1.3.0.9006") + test_that("formula interface", { skip_on_cran() expect_snapshot({ @@ -125,9 +127,11 @@ test_that("unfinalized parameters", { add_model(model_rf) |> add_recipe(rec_example) - set.seed(30) - rf_res <- wf_rf |> - tune_grid(resamples = bt, grid = 4) + expect_message({ + set.seed(30) + rf_res <- wf_rf |> + tune_grid(resamples = bt, grid = 4) + }) expect_snapshot({ set.seed(40) diff --git a/tests/testthat/test-win-loss-overall.R b/tests/testthat/test-win-loss-overall.R index e625a3b..0ad2377 100644 --- a/tests/testthat/test-win-loss-overall.R +++ b/tests/testthat/test-win-loss-overall.R @@ -80,7 +80,7 @@ test_that("one player is really bad", { df_folds <- vfold_cv(df, strata = y) rf_spec <- - rand_forest(min_n = tune(), trees = 10) |> + rand_forest(trees = tune()) |> set_engine("ranger") |> set_mode("classification") @@ -88,7 +88,7 @@ test_that("one player is really bad", { add_formula(y ~ .) |> add_model(rf_spec) - grid <- tibble(min_n = c(1, 40)) + grid <- tibble(trees = c(1, 200)) ctrl <- control_race(burn_in = 2, alpha = .05, randomize = TRUE) set.seed(3355) tuning_results <- tune_race_win_loss(