Post ICML Refactor (part 1) (#257)

* Various improvements

* Refactor website_format.py

* Add patience_callback

* Add logic to TabArena class

* Add generic metric impact evaluation to TabArena class

Author: Innixma

bencheval/bencheval/tabarena.py

@@ -2,6 +2,9 @@
 
 import copy
 import os
+from dataclasses import dataclass
+from typing import Callable, Literal
+from collections.abc import Iterable
 from pathlib import Path
 
 import numpy as np
@@ -18,6 +21,32 @@
 IMPROVABILITY = "improvability"
 LOSS_RESCALED = "loss_rescaled"
 
+MetricDirection = Literal["min", "max"]
+MetricAlignment = Literal["row", "method"]
+InvalidSubsetPolicy = Literal["raise", "nan", "skip"]
+
+
+@dataclass(frozen=True, slots=True)
+class MetricSpec:
+    """
+    Defines how to (re)compute a metric from a subset of results_per_task and how to
+    reduce it to a single scalar score for a given method.
+
+    - compute(): returns either
+        * row-aligned Series (index == results_per_task.index)  [alignment="row"]
+        * method-aligned Series (index == method names)         [alignment="method"]
+    - score(): returns a float score for method_1 given the computed metric result
+    """
+    name: str
+    direction: MetricDirection
+    alignment: MetricAlignment
+    compute: Callable[["TabArena", pd.DataFrame], pd.Series]
+    score: Callable[["TabArena", pd.DataFrame, pd.Series, str], float]
+    # Methods that must be present in any subset (e.g., Elo calibration framework)
+    required_methods: frozenset[str] = frozenset()
+    # What to do if required methods are missing from a subset
+    invalid_subset_policy: InvalidSubsetPolicy = "raise"
+
 
 # TODO: Should "data" be an init arg? Probably not.
 class TabArena:
@@ -420,6 +449,12 @@ def fillna_data(
 
         return data
 
+    def get_task_groupby_cols(self, include_seed_col: bool = False):
+        task_groupby_cols = self.task_groupby_columns
+        if include_seed_col and self.seed_column is not None:
+            task_groupby_cols = task_groupby_cols + [self.seed_column]
+        return task_groupby_cols
+
     def compute_results_per_task(self, data: pd.DataFrame, include_seed_col: bool = False) -> pd.DataFrame:
         groupby_cols = self.groupby_columns
         task_groupby_cols = self.task_groupby_columns
@@ -996,6 +1031,316 @@ def _weighted_groupby_mean(self, tasks: list[str], data: pd.DataFrame, agg_colum
         column_mean.index.name = agg_column
         return column_mean
 
+    def _seed_col_if_present(self, df: pd.DataFrame) -> str | None:
+        if self.seed_column is not None and self.seed_column in df.columns:
+            return self.seed_column
+        return None
+
+    def _score_weighted_mean_by_task(
+        self,
+        df: pd.DataFrame,
+        *,
+        value_col: str,
+        sort_asc: bool,
+    ) -> pd.Series:
+        """
+        Returns a per-method Series of weighted means using the same equal-task weighting
+        logic as other parts of TabArena.
+        """
+        seed_col = self._seed_col_if_present(df)
+        return compute_weighted_mean_by_task(
+            df=df,
+            value_col=value_col,
+            task_col=self.task_groupby_columns,
+            seed_col=seed_col,
+            method_col=self.method_col,
+            sort_asc=sort_asc,
+        )
+
+    def score_if_remove_method(
+        self,
+        metric: MetricSpec,
+        results_per_task: pd.DataFrame,
+        *,
+        method_1: str,
+        method_2: str,
+    ) -> float:
+        """
+        Compute the scalar score for method_1 after removing method_2 and recomputing metric.
+        Returns the resulting score (NOT delta).
+        """
+        # Keep your prior convention: if we remove method_1 itself, return baseline score on provided df.
+        if method_1 == method_2:
+            if not self._metric_subset_ok(metric, results_per_task):
+                return float("nan")
+            metric_values = metric.compute(self, results_per_task)
+            return float(metric.score(self, results_per_task, metric_values, method_1))
+
+        subset = results_per_task.loc[results_per_task[self.method_col] != method_2].copy()
+        if not self._metric_subset_ok(metric, subset):
+            return float("nan")
+        metric_values = metric.compute(self, subset)
+        return float(metric.score(self, subset, metric_values, method_1))
+
+    def score_series_if_remove_each_method(
+        self,
+        metric: MetricSpec,
+        results_per_task: pd.DataFrame,
+        *,
+        method_1: str,
+    ) -> pd.Series:
+        """
+        For a fixed method_1, return a Series indexed by method_2 with values = resulting score
+        for method_1 if method_2 were removed.
+        """
+        methods = (
+            results_per_task[self.method_col]
+            .dropna()
+            .astype(str)
+            .unique()
+            .tolist()
+        )
+
+        scores: dict[str, float] = {}
+        for method_2 in methods:
+            # Never propose removing required methods (e.g., Elo calibration framework)
+            if method_2 in metric.required_methods:
+                continue
+            scores[method_2] = self.score_if_remove_method(
+                metric,
+                results_per_task,
+                method_1=method_1,
+                method_2=method_2,
+            )
+
+        s = pd.Series(scores, name=f"{metric.name}_score_for_{method_1}_if_remove_method")
+        # Sorting: for min-metrics ascending is "better"; for max-metrics descending is "better"
+        return s.sort_values(ascending=(metric.direction == "min"))
+
+    def greedy_remove_methods_optimize_score(
+        self,
+        metric: MetricSpec,
+        results_per_task: pd.DataFrame,
+        *,
+        method_1: str,
+        stop_at_score: float | None = None,
+    ) -> pd.Series:
+        """
+        Iteratively remove method_2 that yields the best improvement for method_1
+        according to metric.direction, recomputing the metric each iteration.
+
+        Returns:
+            pd.Series indexed by removed method_2 in removal order
+            values = resulting score for method_1 at that iteration (NOT delta).
+        """
+        current = results_per_task
+        removed_in_order: dict[str, float] = {}
+
+        while True:
+            # Compute current score for method_1 (and stop checks)
+            if not self._metric_subset_ok(metric, current):
+                break
+            current_metric = metric.compute(self, current)
+            cur_score = float(metric.score(self, current, current_metric, method_1))
+
+            # Stop criteria
+            if pd.isna(cur_score):
+                break
+            if stop_at_score is not None:
+                if metric.direction == "min" and cur_score <= stop_at_score:
+                    break
+                if metric.direction == "max" and cur_score >= stop_at_score:
+                    break
+
+            remaining_methods = current[self.method_col].dropna().astype(str).unique().tolist()
+            # Exclude method_1 and any required methods (e.g., calibration framework)
+            candidates = [
+                m for m in remaining_methods
+                if m != method_1 and m not in metric.required_methods
+            ]
+            if not candidates:
+                break
+
+            candidate_scores: dict[str, float] = {}
+            for method_2 in candidates:
+                subset = current.loc[current[self.method_col] != method_2].copy()
+                if not self._metric_subset_ok(metric, subset):
+                    if metric.invalid_subset_policy == "skip":
+                        continue
+                    candidate_scores[method_2] = float("nan")
+                    continue
+                subset_metric = metric.compute(self, subset)
+                candidate_scores[method_2] = float(metric.score(self, subset, subset_metric, method_1))
+
+            scores_s = pd.Series(candidate_scores).dropna()
+            if scores_s.empty:
+                break
+
+            # Choose best candidate depending on direction
+            if metric.direction == "min":
+                best_method_2 = scores_s.idxmin()
+            else:
+                best_method_2 = scores_s.idxmax()
+
+            best_score = float(scores_s.loc[best_method_2])
+            removed_in_order[best_method_2] = best_score
+
+            # Remove best_method_2 and continue
+            current = current.loc[current[self.method_col] != best_method_2]
+
+        return pd.Series(removed_in_order, name=f"{metric.name}_score_iter_for_{method_1}")
+
+    def greedy_score_matrix(
+        self,
+        metric: MetricSpec,
+        results_per_task: pd.DataFrame,
+        *,
+        methods_1: Iterable[str] | None = None,
+        stop_at_score: float | None = None,
+    ) -> pd.DataFrame:
+        """
+        Build a DataFrame:
+          rows = method_2 (removed)
+          cols = method_1
+          cell = resulting score for method_1 at the iteration when method_2 was removed
+        """
+        if methods_1 is None:
+            methods_1 = (
+                results_per_task[self.method_col].dropna().astype(str).unique().tolist()
+            )
+
+        col_series: dict[str, pd.Series] = {}
+        for method_1 in methods_1:
+            col_series[method_1] = self.greedy_remove_methods_optimize_score(
+                metric,
+                results_per_task,
+                method_1=method_1,
+                stop_at_score=stop_at_score,
+            )
+
+        return pd.DataFrame(col_series)
+
+    # ----------------------------
+    # MetricSpec factories
+    # ----------------------------
+
+    def metric_spec_error(self) -> MetricSpec:
+        """
+        Lower is better. Score = weighted mean error (equal task weighting).
+        """
+        def compute(self: "TabArena", df: pd.DataFrame) -> pd.Series:
+            # row-aligned; no recomputation needed
+            return df[self.error_col]
+
+        def score(self: "TabArena", df: pd.DataFrame, values: pd.Series, method_1: str) -> float:
+            groupby_columns = self._get_groupby_cols(df)
+            tmp = df[groupby_columns].copy()
+            tmp[self.error_col] = values.to_numpy()
+            per_method = self._score_weighted_mean_by_task(tmp, value_col=self.error_col, sort_asc=True)
+            return float(per_method.get(method_1, float("nan")))
+
+        return MetricSpec(
+            name=self.error_col,
+            direction="min",
+            alignment="row",
+            compute=compute,
+            score=score,
+        )
+
+    def metric_spec_rank(self) -> MetricSpec:
+        """
+        Lower is better. Score = weighted mean rank.
+        """
+        def compute(self: "TabArena", df: pd.DataFrame) -> pd.Series:
+            task_groupby_cols = self._get_task_groupby_cols(results=df)
+            return self.compare_rank_per(df=df, task_groupby_cols=task_groupby_cols)
+
+        def score(self: "TabArena", df: pd.DataFrame, values: pd.Series, method_1: str) -> float:
+            groupby_columns = self._get_groupby_cols(df)
+            tmp = df[groupby_columns].copy()
+            tmp[RANK] = values.to_numpy()
+            per_method = self._score_weighted_mean_by_task(tmp, value_col=RANK, sort_asc=True)
+            return float(per_method.get(method_1, float("nan")))
+
+        return MetricSpec(
+            name=RANK,
+            direction="min",
+            alignment="row",
+            compute=compute,
+            score=score,
+        )
+
+    def metric_spec_improvability(self) -> MetricSpec:
+        """
+        Lower is better (0 is ideal). Score = weighted mean improvability.
+        """
+        def compute(self: "TabArena", df: pd.DataFrame) -> pd.Series:
+            task_groupby_cols = self._get_task_groupby_cols(results=df)
+            return self.compute_improvability_per(results_per_task=df, task_groupby_cols=task_groupby_cols)
+
+        def score(self: "TabArena", df: pd.DataFrame, values: pd.Series, method_1: str) -> float:
+            groupby_columns = self._get_groupby_cols(df)
+            tmp = df[groupby_columns].copy()
+            tmp[IMPROVABILITY] = values.to_numpy()
+            per_method = self._score_weighted_mean_by_task(tmp, value_col=IMPROVABILITY, sort_asc=True)
+            return float(per_method.get(method_1, float("nan")))
+
+        return MetricSpec(
+            name=IMPROVABILITY,
+            direction="min",
+            alignment="row",
+            compute=compute,
+            score=score,
+        )
+
+    def metric_spec_elo(self, **elo_kwargs) -> MetricSpec:
+        """
+        Higher is better. Score = Elo value for method_1 computed on the subset.
+        """
+        calibration_framework = elo_kwargs.get("calibration_framework", None)
+        required = frozenset([calibration_framework]) if calibration_framework else frozenset()
+
+        def compute(self: "TabArena", df: pd.DataFrame) -> pd.Series:
+            bars = self.compute_elo(
+                results_per_task=df,
+                include_quantiles=False,
+                round_decimals=None,
+                **elo_kwargs,
+            )
+            # method-aligned Series
+            return bars["elo"]
+
+        def score(self: "TabArena", df: pd.DataFrame, values: pd.Series, method_1: str) -> float:
+            return float(values.get(method_1, float("nan")))
+
+        return MetricSpec(
+            name="elo",
+            direction="max",
+            alignment="method",
+            compute=compute,
+            score=score,
+            required_methods=required,
+            invalid_subset_policy="raise",
+        )
+
+    def _metric_subset_ok(self, metric: MetricSpec, df: pd.DataFrame) -> bool:
+        """Return True if df satisfies metric.required_methods; otherwise obey policy."""
+        if not metric.required_methods:
+            return True
+        present = set(df[self.method_col].dropna().astype(str).unique())
+        missing = set(metric.required_methods) - present
+        if not missing:
+            return True
+        if metric.invalid_subset_policy == "raise":
+            raise ValueError(
+                f"Metric {metric.name!r} requires methods {sorted(metric.required_methods)}, "
+                f"but subset is missing {sorted(missing)}."
+            )
+        if metric.invalid_subset_policy == "nan":
+            return False
+        # "skip"
+        return False
+
 
 def get_bootstrap_result_lst(data: list, func_, rng=None, num_round: int = None, func_kwargs=None, seed: int = 0):
     rows = []

examples/benchmarking/custom_tabarena_model/run_evaluate_model.py

@@ -8,7 +8,7 @@
 from pathlib import Path
 
 from tabarena.nips2025_utils.end_to_end_single import EndToEndResultsSingle, EndToEndSingle
-from bencheval.website_format import format_leaderboard
+from tabarena.website.website_format import format_leaderboard
 
 if __name__ == "__main__":
     path_raw = Path(__file__).parent / "tabarena_out" / "custom_model"

examples/benchmarking/run_quickstart_tabarena.py

@@ -8,7 +8,7 @@
 from tabarena.benchmark.experiment import AGModelBagExperiment, ExperimentBatchRunner
 from tabarena.nips2025_utils.end_to_end import EndToEnd
 from tabarena.nips2025_utils.tabarena_context import TabArenaContext
-from bencheval.website_format import format_leaderboard
+from tabarena.website.website_format import format_leaderboard
 
 
 if __name__ == '__main__':

examples/benchmarking/run_quickstart_tabarena_custom_datasets.py

@@ -12,7 +12,7 @@
 from tabarena.models.utils import get_configs_generator_from_name
 from tabarena.nips2025_utils.compare import compare
 from tabarena.nips2025_utils.end_to_end import EndToEnd
-from bencheval.website_format import format_leaderboard
+from tabarena.website.website_format import format_leaderboard
 
 
 def get_custom_classification_task(task_cache_dir: str) -> UserTask: