Update to average_seeds=False by default in all places

Author: Innixma

bencheval/bencheval/tabarena.py

@@ -107,7 +107,7 @@ def _get_groupby_cols(self, results: pd.DataFrame) -> list[str]:
     def leaderboard(
         self,
         data: pd.DataFrame,
-        average_seeds: bool = True,
+        average_seeds: bool = False,
         include_error: bool = False,
         include_elo: bool = True,
         include_winrate: bool = True,

tabarena/tabarena/nips2025_utils/compare.py

@@ -20,7 +20,7 @@ def compare_on_tabarena(
     tabarena_context_kwargs: dict | None = None,
     fillna: str | pd.DataFrame | None = "RF (default)",
     score_on_val: bool = False,
-    average_seeds: bool = True,
+    average_seeds: bool = False,
     remove_imputed: bool = False,
     tmp_treat_tasks_independently: bool = False,
     leaderboard_kwargs: dict | None = None,
@@ -103,7 +103,7 @@ def compare(
     calibration_framework: str | None = None,
     fillna: str | pd.DataFrame | None = None,
     score_on_val: bool = False,
-    average_seeds: bool = True,
+    average_seeds: bool = False,
     tmp_treat_tasks_independently: bool = False,  # FIXME: Update
     leaderboard_kwargs: dict | None = None,
     remove_imputed: bool = False,

tabarena/tabarena/nips2025_utils/end_to_end.py

@@ -383,7 +383,7 @@ def compare(
         use_artifact_name_in_prefix: bool | None = None,
         use_model_results: bool = False,
         score_on_val: bool = False,
-        average_seeds: bool = True,
+        average_seeds: bool = False,
         leaderboard_kwargs: dict | None = None,
     ):
         results = self.get_results(
@@ -414,7 +414,7 @@ def compare_on_tabarena(
         use_artifact_name_in_prefix: bool | None = None,
         use_model_results: bool = False,
         score_on_val: bool = False,
-        average_seeds: bool = True,
+        average_seeds: bool = False,
         leaderboard_kwargs: dict | None = None,
         tabarena_context_kwargs: dict | None = None,
         extra_results: pd.DataFrame = None,

tabarena/tabarena/nips2025_utils/end_to_end_single.py

@@ -566,7 +566,7 @@ def compare_on_tabarena(
         use_artifact_name_in_prefix: bool | None = None,
         use_model_results: bool = False,
         score_on_val: bool = False,
-        average_seeds: bool = True,
+        average_seeds: bool = False,
         leaderboard_kwargs: dict | None = None,
         extra_results: pd.DataFrame = None,
         tabarena_context_kwargs: dict = None,

tabarena/tabarena/nips2025_utils/tabarena_context.py

@@ -114,7 +114,7 @@ def compare(
         subset: str | list[str] | None = None,
         folds: list[int] | None = None,
         score_on_val: bool = False,
-        average_seeds: bool = True,
+        average_seeds: bool = False,
         fillna: str | pd.DataFrame | None = "RF (default)",
         remove_imputed: bool = False,
         tmp_treat_tasks_independently: bool = False,
@@ -386,7 +386,7 @@ def simulate_portfolio_search(
         n_portfolio: int = 25,
         n_ensemble: int = 40,
         time_limit: float | None = 14400,
-        average_seeds: bool = True,
+        average_seeds: bool = False,
     ):
         if repo is None:
             repo = self.load_repo(methods=methods, config_fallback=config_fallback)

tabarena/tabarena/paper/paper_runner_tabarena.py

@@ -16,7 +16,7 @@ def run_portfolio_search(
         n_portfolio: int = 25,
         n_ensemble: int = 40,
         time_limit: float | None = 14400,
-        average_seeds: bool = True,
+        average_seeds: bool = False,
     ) -> pd.DataFrame:
         calibration_framework = "RF (default)"
         elo_bootstrap_rounds = 100

tabarena/tabarena/paper/tabarena_evaluator.py

@@ -229,7 +229,7 @@ def eval(
         plot_pareto: bool = True,
         plot_other: bool = False,
         calibration_framework: str | None = "auto",
-        average_seeds: bool = True,
+        average_seeds: bool = False,
         tmp_treat_tasks_independently: bool = False,  # FIXME: Need to make a weighted elo logic
         leaderboard_kwargs: dict | None = None,
         plot_with_baselines: bool = False,