fix: ensure that display_on_leaderboard actually reflect whether the benchmark is displayed (#4288)

* fix: ensure that `display_on_leaderboard` actually reflect whether the benchmark is displayed

I believe the previous attribute was a leftover from an earlier version of the leaderboard

* fix typing

Author: KennethEnevoldsen

mteb/benchmarks/benchmark.py

@@ -2,6 +2,7 @@
 
 from collections.abc import Iterator, Sequence
 from dataclasses import dataclass, field
+from functools import lru_cache
 from typing import TYPE_CHECKING, Literal, cast
 
 import huggingface_hub
@@ -15,6 +16,32 @@
     from mteb.results import BenchmarkResults
 
 
+@lru_cache
+def _get_benchmarks_on_leaderboard() -> set[str]:
+    from mteb.leaderboard.benchmark_selector import (
+        GP_BENCHMARK_ENTRIES,
+        R_BENCHMARK_ENTRIES,
+        MenuEntry,
+    )
+
+    entries = GP_BENCHMARK_ENTRIES + R_BENCHMARK_ENTRIES
+
+    def __extract_benchmarks(
+        entries: Sequence[Benchmark | MenuEntry],
+    ) -> list[Benchmark]:
+        benchmarks = []
+        for entry in entries:
+            if isinstance(entry, Benchmark):
+                benchmarks.append(entry)
+            else:
+                benchmarks.extend(__extract_benchmarks(entry.benchmarks))
+        return benchmarks
+
+    names = {benchmark.name for benchmark in __extract_benchmarks(entries)}
+
+    return names
+
+
 @dataclass
 class Benchmark:
     """A benchmark object intended to run a certain benchmark within MTEB.
@@ -46,11 +73,16 @@ class Benchmark:
     reference: StrURL | None = None
     citation: str | None = None
     contacts: list[str] | None = None
-    display_on_leaderboard: bool = True
     icon: str | None = None
     display_name: str | None = None
     language_view: list[str] | Literal["all"] = field(default_factory=list)
 
+    @property
+    def display_on_leaderboard(self) -> bool:
+        """Whether the benchmark should be displayed on the leaderboard."""
+        benchmarks_on_leaderboard = _get_benchmarks_on_leaderboard()
+        return self.name in benchmarks_on_leaderboard
+
     def __iter__(self) -> Iterator[AbsTask]:
         return iter(self.tasks)
 

tests/test_benchmarks/test_get_benchmarks.py

@@ -31,3 +31,17 @@ def test_benchmark_aliases(alias, full_name):
     assert benchmark.name == full_name
     assert isinstance(benchmark, mteb.Benchmark)
     assert alias in benchmark.aliases
+
+
+def test_benchmark_on_leaderboard():
+    on_leaderboard = "MTEB(Multilingual, v2)"
+    not_on_leaderboard = "MTEB(Multilingual, v1)"
+    benchmark = mteb.get_benchmarks(display_on_leaderboard=True)
+    names = {b.name for b in benchmark}
+    assert on_leaderboard in names
+    assert not_on_leaderboard not in names
+
+    benchmark = mteb.get_benchmarks(display_on_leaderboard=False)
+    names = {b.name for b in benchmark}
+    assert on_leaderboard not in names
+    assert not_on_leaderboard in names