diff --git a/R/tune_race_anova.R b/R/tune_race_anova.R index 0f0c4f1..1db1844 100644 --- a/R/tune_race_anova.R +++ b/R/tune_race_anova.R @@ -19,7 +19,8 @@ #' data frame should have columns for each parameter being tuned and rows for #' tuning parameter candidates. An integer denotes the number of candidate #' parameter sets to be created automatically. -#' @param metrics A [yardstick::metric_set()] or `NULL`. +#' @param metrics A [yardstick::metric_set()] or `NULL`. The first metric in +#' metrics is the one that will be optimized. #' @param eval_time A numeric vector of time points where dynamic event time #' metrics should be computed (e.g. the time-dependent ROC curve, etc). The #' values must be non-negative and should probably be no greater than the diff --git a/R/tune_race_win_loss.R b/R/tune_race_win_loss.R index f8ca9d3..d8b06bf 100644 --- a/R/tune_race_win_loss.R +++ b/R/tune_race_win_loss.R @@ -51,8 +51,9 @@ #' statistical analysis is updated. More candidate parameters may be excluded #' with each new resample that is processed. #' -#' The [control_race()] function contains are parameter for the significance cutoff +#' The [control_race()] function contains parameters for the significance cutoff #' applied to the Bradley-Terry model results as well as other relevant arguments. +#' The first metric in the `metrics` argument is the one that will be optimized. #' #' ## Censored regression models #' diff --git a/tests/testthat/_snaps/sa-misc.md b/tests/testthat/_snaps/sa-misc.md index 52df4bf..3dce56b 100644 --- a/tests/testthat/_snaps/sa-misc.md +++ b/tests/testthat/_snaps/sa-misc.md @@ -6,9 +6,9 @@ Message Optimizing roc_auc Initial best: 0.85731 - 1 ( ) accept suboptimal roc_auc=0.85682 (+/-0.01022) - 2 ( ) accept suboptimal roc_auc=0.85238 (+/-0.01078) - 3 ( ) accept suboptimal roc_auc=0.85138 (+/-0.0109) + 1 ( ) accept suboptimal roc_auc=0.85432 (+/-0.01039) + 2 <3 new best roc_auc=0.86674 (+/-0.009097) + 3 <3 new best roc_auc=0.87245 (+/-0.008363) --- @@ -18,9 +18,9 @@ Message Optimizing roc_auc Initial best: 0.85325 - 1 ( ) accept suboptimal roc_auc=0.85313 (+/-0.0106) - 2 ( ) accept suboptimal roc_auc=0.85181 (+/-0.01065) - 3 ( ) accept suboptimal roc_auc=0.85165 (+/-0.01055) + 1 ( ) accept suboptimal roc_auc=0.85268 (+/-0.01056) + 2 <3 new best roc_auc=0.85618 (+/-0.01015) + 3 <3 new best roc_auc=0.85799 (+/-0.009991) --- @@ -30,9 +30,9 @@ Message Optimizing roc_auc Initial best: 0.86616 - 1 ( ) accept suboptimal roc_auc=0.86399 (+/-0.01081) - 2 <3 new best roc_auc=0.86768 (+/-0.009563) - 3 <3 new best roc_auc=0.87329 (+/-0.008273) + 1 ( ) accept suboptimal roc_auc=0.86568 (+/-0.01027) + 2 <3 new best roc_auc=0.86654 (+/-0.009714) + 3 <3 new best roc_auc=0.8688 (+/-0.009487) --- @@ -42,7 +42,15 @@ Message Optimizing roc_auc Initial best: 0.86616 - 1 ( ) accept suboptimal roc_auc=0.86399 (+/-0.01081) - 2 <3 new best roc_auc=0.86768 (+/-0.009563) - 3 <3 new best roc_auc=0.87329 (+/-0.008273) + 1 ( ) accept suboptimal roc_auc=0.86568 (+/-0.01027) + 2 <3 new best roc_auc=0.86654 (+/-0.009714) + 3 <3 new best roc_auc=0.8688 (+/-0.009487) + +# tune_sim_anneal with wrong type + + Code + tune_sim_anneal(1) + Condition + Error in `tune_sim_anneal()`: + ! The first argument to `tune_sim_anneal()` should be either a model or workflow. diff --git a/tests/testthat/_snaps/sa-overall.md b/tests/testthat/_snaps/sa-overall.md index 4e1b77b..aead642 100644 --- a/tests/testthat/_snaps/sa-overall.md +++ b/tests/testthat/_snaps/sa-overall.md @@ -12,79 +12,43 @@ Optimizing roc_auc Initial best: 0.73008 i Fold1, Repeat1: preprocessor 1/1 - v Fold1, Repeat1: preprocessor 1/1 i Fold1, Repeat1: preprocessor 1/1, model 1/1 - v Fold1, Repeat1: preprocessor 1/1, model 1/1 - i Fold1, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat1: preprocessor 1/1 - v Fold2, Repeat1: preprocessor 1/1 i Fold2, Repeat1: preprocessor 1/1, model 1/1 - v Fold2, Repeat1: preprocessor 1/1, model 1/1 - i Fold2, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat1: preprocessor 1/1 - v Fold3, Repeat1: preprocessor 1/1 i Fold3, Repeat1: preprocessor 1/1, model 1/1 - v Fold3, Repeat1: preprocessor 1/1, model 1/1 - i Fold3, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold1, Repeat2: preprocessor 1/1 - v Fold1, Repeat2: preprocessor 1/1 i Fold1, Repeat2: preprocessor 1/1, model 1/1 - v Fold1, Repeat2: preprocessor 1/1, model 1/1 - i Fold1, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat2: preprocessor 1/1 - v Fold2, Repeat2: preprocessor 1/1 i Fold2, Repeat2: preprocessor 1/1, model 1/1 - v Fold2, Repeat2: preprocessor 1/1, model 1/1 - i Fold2, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat2: preprocessor 1/1 - v Fold3, Repeat2: preprocessor 1/1 i Fold3, Repeat2: preprocessor 1/1, model 1/1 - v Fold3, Repeat2: preprocessor 1/1, model 1/1 - i Fold3, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat2: preprocessor 1/1, model 1/1 (predictions) - 1 ( ) accept suboptimal roc_auc=0.72145 (+/-0.003605) + 1 <3 new best roc_auc=0.74036 (+/-0.002433) i Fold1, Repeat1: preprocessor 1/1 - v Fold1, Repeat1: preprocessor 1/1 i Fold1, Repeat1: preprocessor 1/1, model 1/1 - v Fold1, Repeat1: preprocessor 1/1, model 1/1 - i Fold1, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat1: preprocessor 1/1 - v Fold2, Repeat1: preprocessor 1/1 i Fold2, Repeat1: preprocessor 1/1, model 1/1 - v Fold2, Repeat1: preprocessor 1/1, model 1/1 - i Fold2, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat1: preprocessor 1/1 - v Fold3, Repeat1: preprocessor 1/1 i Fold3, Repeat1: preprocessor 1/1, model 1/1 - v Fold3, Repeat1: preprocessor 1/1, model 1/1 - i Fold3, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold1, Repeat2: preprocessor 1/1 - v Fold1, Repeat2: preprocessor 1/1 i Fold1, Repeat2: preprocessor 1/1, model 1/1 - v Fold1, Repeat2: preprocessor 1/1, model 1/1 - i Fold1, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat2: preprocessor 1/1 - v Fold2, Repeat2: preprocessor 1/1 i Fold2, Repeat2: preprocessor 1/1, model 1/1 - v Fold2, Repeat2: preprocessor 1/1, model 1/1 - i Fold2, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat2: preprocessor 1/1 - v Fold3, Repeat2: preprocessor 1/1 i Fold3, Repeat2: preprocessor 1/1, model 1/1 - v Fold3, Repeat2: preprocessor 1/1, model 1/1 - i Fold3, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat2: preprocessor 1/1, model 1/1 (predictions) - 2 <3 new best roc_auc=0.73173 (+/-0.003018) + 2 ( ) accept suboptimal roc_auc=0.73543 (+/-0.009624) # variable interface @@ -100,79 +64,43 @@ Optimizing roc_auc Initial best: 0.73008 i Fold1, Repeat1: preprocessor 1/1 - v Fold1, Repeat1: preprocessor 1/1 i Fold1, Repeat1: preprocessor 1/1, model 1/1 - v Fold1, Repeat1: preprocessor 1/1, model 1/1 - i Fold1, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat1: preprocessor 1/1 - v Fold2, Repeat1: preprocessor 1/1 i Fold2, Repeat1: preprocessor 1/1, model 1/1 - v Fold2, Repeat1: preprocessor 1/1, model 1/1 - i Fold2, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat1: preprocessor 1/1 - v Fold3, Repeat1: preprocessor 1/1 i Fold3, Repeat1: preprocessor 1/1, model 1/1 - v Fold3, Repeat1: preprocessor 1/1, model 1/1 - i Fold3, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold1, Repeat2: preprocessor 1/1 - v Fold1, Repeat2: preprocessor 1/1 i Fold1, Repeat2: preprocessor 1/1, model 1/1 - v Fold1, Repeat2: preprocessor 1/1, model 1/1 - i Fold1, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat2: preprocessor 1/1 - v Fold2, Repeat2: preprocessor 1/1 i Fold2, Repeat2: preprocessor 1/1, model 1/1 - v Fold2, Repeat2: preprocessor 1/1, model 1/1 - i Fold2, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat2: preprocessor 1/1 - v Fold3, Repeat2: preprocessor 1/1 i Fold3, Repeat2: preprocessor 1/1, model 1/1 - v Fold3, Repeat2: preprocessor 1/1, model 1/1 - i Fold3, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat2: preprocessor 1/1, model 1/1 (predictions) - 1 ( ) accept suboptimal roc_auc=0.72145 (+/-0.003605) + 1 <3 new best roc_auc=0.74036 (+/-0.002433) i Fold1, Repeat1: preprocessor 1/1 - v Fold1, Repeat1: preprocessor 1/1 i Fold1, Repeat1: preprocessor 1/1, model 1/1 - v Fold1, Repeat1: preprocessor 1/1, model 1/1 - i Fold1, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat1: preprocessor 1/1 - v Fold2, Repeat1: preprocessor 1/1 i Fold2, Repeat1: preprocessor 1/1, model 1/1 - v Fold2, Repeat1: preprocessor 1/1, model 1/1 - i Fold2, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat1: preprocessor 1/1 - v Fold3, Repeat1: preprocessor 1/1 i Fold3, Repeat1: preprocessor 1/1, model 1/1 - v Fold3, Repeat1: preprocessor 1/1, model 1/1 - i Fold3, Repeat1: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat1: preprocessor 1/1, model 1/1 (predictions) i Fold1, Repeat2: preprocessor 1/1 - v Fold1, Repeat2: preprocessor 1/1 i Fold1, Repeat2: preprocessor 1/1, model 1/1 - v Fold1, Repeat2: preprocessor 1/1, model 1/1 - i Fold1, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold1, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold2, Repeat2: preprocessor 1/1 - v Fold2, Repeat2: preprocessor 1/1 i Fold2, Repeat2: preprocessor 1/1, model 1/1 - v Fold2, Repeat2: preprocessor 1/1, model 1/1 - i Fold2, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold2, Repeat2: preprocessor 1/1, model 1/1 (predictions) i Fold3, Repeat2: preprocessor 1/1 - v Fold3, Repeat2: preprocessor 1/1 i Fold3, Repeat2: preprocessor 1/1, model 1/1 - v Fold3, Repeat2: preprocessor 1/1, model 1/1 - i Fold3, Repeat2: preprocessor 1/1, model 1/1 (extracts) i Fold3, Repeat2: preprocessor 1/1, model 1/1 (predictions) - 2 <3 new best roc_auc=0.73173 (+/-0.003018) + 2 ( ) accept suboptimal roc_auc=0.73543 (+/-0.009624) --- @@ -183,9 +111,9 @@ Message There were 2 previous iterations Optimizing roc_auc - 2 v initial roc_auc=0.73173 (+/-0.003018) - 3 <3 new best roc_auc=0.74172 (+/-0.008775) - 4 <3 new best roc_auc=0.7909 (+/-0.009887) + 2 v initial roc_auc=0.74036 (+/-0.002433) + 3 <3 new best roc_auc=0.75103 (+/-0.009282) + 4 <3 new best roc_auc=0.76617 (+/-0.01037) --- @@ -205,19 +133,19 @@ set.seed(40) rf_res_finetune <- tune_sim_anneal(wf_rf, resamples = bt, initial = rf_res) Message - i Creating pre-processing data to finalize unknown parameter: mtry + i Creating pre-processing data to finalize 1 unknown parameter: "mtry" Optimizing roc_auc - Initial best: 0.84994 - 1 ( ) accept suboptimal roc_auc=0.84375 (+/-0.007727) - 2 + better suboptimal roc_auc=0.84943 (+/-0.007036) - 3 ( ) accept suboptimal roc_auc=0.84371 (+/-0.007903) - 4 + better suboptimal roc_auc=0.84825 (+/-0.008036) - 5 ( ) accept suboptimal roc_auc=0.84479 (+/-0.00814) - 6 + better suboptimal roc_auc=0.84816 (+/-0.007283) - 7 ( ) accept suboptimal roc_auc=0.84381 (+/-0.007999) - 8 <3 new best roc_auc=0.85014 (+/-0.007172) - 9 ( ) accept suboptimal roc_auc=0.84344 (+/-0.007818) - 10 + better suboptimal roc_auc=0.84802 (+/-0.007281) + Initial best: 0.84917 + 1 ( ) accept suboptimal roc_auc=0.84467 (+/-0.007843) + 2 <3 new best roc_auc=0.84956 (+/-0.007013) + 3 ( ) accept suboptimal roc_auc=0.84384 (+/-0.008042) + 4 + better suboptimal roc_auc=0.84952 (+/-0.00757) + 5 ( ) accept suboptimal roc_auc=0.84357 (+/-0.008148) + 6 + better suboptimal roc_auc=0.84848 (+/-0.007499) + 7 ( ) accept suboptimal roc_auc=0.84391 (+/-0.008097) + 8 + better suboptimal roc_auc=0.84853 (+/-0.007036) + 9 ( ) accept suboptimal roc_auc=0.84471 (+/-0.007812) + 10 x restart from best roc_auc=0.84935 (+/-0.007313) --- @@ -225,19 +153,19 @@ set.seed(40) rf_res_finetune <- tune_sim_anneal(wf_rf, resamples = bt) Message - i Creating pre-processing data to finalize unknown parameter: mtry + i Creating pre-processing data to finalize 1 unknown parameter: "mtry" Optimizing roc_auc - Initial best: 0.84418 - 1 <3 new best roc_auc=0.84839 (+/-0.007753) - 2 ( ) accept suboptimal roc_auc=0.84384 (+/-0.008085) - 3 <3 new best roc_auc=0.84857 (+/-0.007615) - 4 ( ) accept suboptimal roc_auc=0.8435 (+/-0.007746) - 5 + better suboptimal roc_auc=0.84804 (+/-0.00774) - 6 ( ) accept suboptimal roc_auc=0.84338 (+/-0.007515) - 7 <3 new best roc_auc=0.84923 (+/-0.007371) - 8 ( ) accept suboptimal roc_auc=0.84389 (+/-0.007938) - 9 <3 new best roc_auc=0.84926 (+/-0.007163) - 10 ( ) accept suboptimal roc_auc=0.84397 (+/-0.00741) + Initial best: 0.84369 + 1 <3 new best roc_auc=0.84929 (+/-0.007889) + 2 ( ) accept suboptimal roc_auc=0.84461 (+/-0.008084) + 3 + better suboptimal roc_auc=0.84785 (+/-0.007989) + 4 ( ) accept suboptimal roc_auc=0.84325 (+/-0.007817) + 5 + better suboptimal roc_auc=0.84917 (+/-0.007376) + 6 ( ) accept suboptimal roc_auc=0.84458 (+/-0.007566) + 7 + better suboptimal roc_auc=0.84871 (+/-0.007483) + 8 ( ) accept suboptimal roc_auc=0.84431 (+/-0.007694) + 9 x restart from best roc_auc=0.84861 (+/-0.007929) + 10 ( ) accept suboptimal roc_auc=0.84522 (+/-0.008113) # incompatible parameter objects diff --git a/tests/testthat/test-win-loss-overall.R b/tests/testthat/test-win-loss-overall.R index e625a3b..8c7506b 100644 --- a/tests/testthat/test-win-loss-overall.R +++ b/tests/testthat/test-win-loss-overall.R @@ -88,7 +88,7 @@ test_that("one player is really bad", { add_formula(y ~ .) |> add_model(rf_spec) - grid <- tibble(min_n = c(1, 40)) + grid <- tibble(min_n = c(1, 100)) ctrl <- control_race(burn_in = 2, alpha = .05, randomize = TRUE) set.seed(3355) tuning_results <- tune_race_win_loss(