LLM names take into account reasoning effort in model args

src/agenteval/leaderboard/model_name_mapping.py

@@ -85,4 +85,6 @@
     "gemini/gemini-2.5-pro": "Gemini 2.5 Pro (unpinned)",
     "openai/gpt-4o": "GPT-4o (unpinned)",
     "gpt-3.5-turbo-0125": "GPT-3.5 Turbo (2025-01)",
+    "openai/gpt-5": "GPT-5 (unpinned)",
+    "gpt-5": "GPT-5 (unpinned)",
 }

src/agenteval/leaderboard/view.py

@@ -16,6 +16,7 @@
 
 from .. import compute_summary_statistics
 from ..config import SuiteConfig
+from ..score import EvalSpec
 from .model_name_mapping import LB_MODEL_NAME_MAPPING
 from .models import LeaderboardSubmission
 
@@ -366,6 +367,79 @@ def construct_reproducibility_url(task_revisions: list[EvalRevision]) -> str | N
     return source_url
 
 
+def adjust_model_name_for_reasoning_effort(model_name: str, effort: str) -> str:
+    return f"{model_name} (reasoning_effort={effort})"
+
+
+def get_model_name_aliases(raw_name: str) -> set[str]:
+    aliases = {raw_name}
+    if raw_name in LB_MODEL_NAME_MAPPING:
+        # pretty just means a value in our LB_MODEL_NAME_MAPPING map
+        pretty_name = LB_MODEL_NAME_MAPPING[raw_name]
+        aliases.add(pretty_name)
+
+        # if the pretty name suggests it's unpinned
+        # include the pretty version without the date part
+        open_paren_index = pretty_name.rindex("(")
+        name_date = pretty_name[open_paren_index:].strip()
+        if name_date == "(unpinned)":
+            dateless_pretty_name = pretty_name[:open_paren_index].strip()
+            aliases.add(dateless_pretty_name)
+    return {a.lower() for a in aliases}
+
+
+def format_model_names_for_one_result(
+    raw_names: set[str], eval_spec: EvalSpec | None
+) -> dict[str, str]:
+    to_return: dict[str, str] = {}
+
+    if (
+        (eval_spec is not None)
+        and (eval_spec.model_args is not None)
+        and (isinstance(eval_spec.model_args, dict))
+        and ("reasoning_effort" in eval_spec.model_args)
+    ):
+        consider_eval_spec = True
+        spec_model_name_aliases = get_model_name_aliases(eval_spec.model)
+    else:
+        consider_eval_spec = False
+        spec_model_name_aliases = None
+
+    for raw_name in raw_names:
+        safe_name_option = LB_MODEL_NAME_MAPPING.get(raw_name, raw_name)
+        other_name_option = None
+
+        if consider_eval_spec:
+            # make mypy happy
+            assert eval_spec is not None
+            assert spec_model_name_aliases is not None
+            assert isinstance(eval_spec.model_args, dict)
+            raw_name_aliases = get_model_name_aliases(raw_name)
+            looks_like_same_model = (
+                len(raw_name_aliases.intersection(spec_model_name_aliases)) > 0
+            )
+            if looks_like_same_model:
+                reasoning_effort = eval_spec.model_args["reasoning_effort"]
+                other_name_option = adjust_model_name_for_reasoning_effort(
+                    model_name=safe_name_option,
+                    effort=reasoning_effort,
+                )
+
+        to_use = safe_name_option if other_name_option is None else other_name_option
+        to_return[raw_name] = to_use
+
+    return to_return
+
+
+def merge_in_formatted_names_from_one_result(
+    so_far: dict[str, set[str]], from_one_result: dict[str, str]
+):
+    for k, v in from_one_result.items():
+        if k not in so_far:
+            so_far[k] = set()
+        so_far[k].add(v)
+
+
 def _get_dataframe(
     eval_results: datasets.DatasetDict,
     split: str,
@@ -397,6 +471,9 @@ def _get_dataframe(
         )
 
         model_token_counts: dict[str, int] = {}
+        # formatted model names
+        raw_names_to_formatted_names: dict[str, set[str]] = {}
+
         if ev.results:
             for task_result in ev.results:
 
@@ -407,6 +484,7 @@ def _get_dataframe(
                     task_result.model_usages = None
                     task_result.model_costs = None
 
+                models_in_this_task = set([])
                 if task_result.model_usages:
                     for usage_list in task_result.model_usages:
                         for model_usage in usage_list:
@@ -418,14 +496,31 @@ def _get_dataframe(
                             else:
                                 model_token_counts[model_name] = total_tokens
 
+                            models_in_this_task.add(model_name)
+
+                merge_in_formatted_names_from_one_result(
+                    so_far=raw_names_to_formatted_names,
+                    from_one_result=format_model_names_for_one_result(
+                        raw_names=models_in_this_task,
+                        eval_spec=task_result.eval_spec,
+                    ),
+                )
+
         # Sort by cumulative token count (descending - most used first)
         sorted_raw_names = sorted(
             model_token_counts.keys(), key=lambda x: model_token_counts[x], reverse=True
         )
 
-        model_names = [
-            LB_MODEL_NAME_MAPPING.get(name, name) for name in sorted_raw_names
-        ]
+        # use a list because order matter here
+        model_names = []
+        for raw_name in sorted_raw_names:
+            # we might have mapped the same raw name to different formatted names
+            # e.g. if reasoning effort wasn't at the default for a result
+            formatted_names = raw_names_to_formatted_names[raw_name]
+            # in case two raw names map to the same formatted name
+            for formatted_name in formatted_names:
+                if formatted_name not in model_names:
+                    model_names.append(formatted_name)
 
         # only format if submit_time present, else leave as None
         ts = sub.submit_time