fix: Added leaderboard Vidore V3 (#3542)

* feat:initial leaderboard proposal

* feat: update summary table for ViDoRe V3 to reflect Document Understanding tasks

* refactor: update leaderboard references

* fix: update VISUAL_DOCUMENT_RETRIEVAL to use VidoreBenchmark

* fix: update JinaVisualDocumentBenchmark summary table creation method

* fix: add VisualDocumentRetrieval to previous benchmark names

* fix: remove JinaVisualDocumentBenchmark

---------

Co-authored-by: Antoine Edy <[email protected]>

Author: QuentinJGMace

mteb/benchmarks/_create_table.py

@@ -358,9 +358,7 @@ def _create_summary_table_mean_public_private(
         "mean(public)": "Mean (Public)",
         "mean(private)": "Mean (Private)",
     }
-    # For RTEB: all tasks are Retrieval type, so Retrieval column = Mean (Task)
-    if "Retrieval" in joint_table.columns:
-        rename_dict["Retrieval"] = "Mean (Task)"
+
     joint_table = joint_table.rename(columns=rename_dict)
 
     # Move borda rank to front

mteb/benchmarks/benchmark.py

@@ -87,7 +87,10 @@ class RtebBenchmark(Benchmark):
     def _create_summary_table(
         self, benchmark_results: BenchmarkResults
     ) -> pd.DataFrame:
-        return _create_summary_table_mean_public_private(benchmark_results)
+        joint_table = _create_summary_table_mean_public_private(benchmark_results)
+        # For RTEB: all tasks are Retrieval type, so Retrieval column = Mean (Task)
+        joint_table = joint_table.rename(columns={"Retrieval": "Mean (Task)"})
+        return joint_table
 
 
 class HUMEBenchmark(Benchmark):
@@ -108,10 +111,15 @@ def _create_summary_table(
         return _create_summary_table_mean_task_type(benchmark_results)
 
 
-class Vidore3Benchmark(Benchmark):
+class VidoreBenchmark(Benchmark):
     """Wrapper for Vidore3 benchmark."""
 
     def _create_summary_table(
         self, benchmark_results: BenchmarkResults
     ) -> pd.DataFrame:
-        return _create_summary_table_mean_public_private(benchmark_results)
+        joint_table = _create_summary_table_mean_public_private(benchmark_results)
+        # For ViDoRe (V1, V2, V3): all tasks are Document Understanding type, so Document Understanding column = Mean (Task)
+        joint_table = joint_table.rename(
+            columns={"Document Understanding": "Mean (Task)"}
+        )
+        return joint_table

mteb/benchmarks/benchmarks/benchmarks.py

@@ -2,7 +2,7 @@
     Benchmark,
     HUMEBenchmark,
     MIEBBenchmark,
-    Vidore3Benchmark,
+    VidoreBenchmark,
 )
 from mteb.get_tasks import MTEBTasks, get_task, get_tasks
 
@@ -2219,7 +2219,7 @@
 """,
 )
 
-VIDORE_V3 = Vidore3Benchmark(
+VIDORE_V3 = VidoreBenchmark(
     name="ViDoRe(v3)",
     display_name="ViDoRe V3",
     icon="https://cdn-uploads.huggingface.co/production/uploads/66e16a677c2eb2da5109fb5c/x99xqw__fl2UaPbiIdC_f.png",
@@ -2253,10 +2253,9 @@
 """,
 )
 
-VISUAL_DOCUMENT_RETRIEVAL = Benchmark(
-    name="VisualDocumentRetrieval",
-    display_name="Visual Document Retrieval",
-    icon="https://github.com/DennisSuitters/LibreICONS/raw/2d2172d15e3c6ca03c018629d60050e4b99e5c55/svg-color/libre-gui-picture.svg",
+VISUAL_DOCUMENT_RETRIEVAL = VidoreBenchmark(
+    name="ViDoRe(v1&v2)",
+    display_name="ViDoRe (V1&V2)",
     tasks=get_tasks(
         tasks=[
             # v1

mteb/benchmarks/get_benchmark.py

@@ -39,6 +39,7 @@ def _get_previous_benchmark_names() -> dict[str, str]:
         MTEB_RETRIEVAL_MEDICAL,
         MTEB_RETRIEVAL_WITH_INSTRUCTIONS,
         SEB,
+        VISUAL_DOCUMENT_RETRIEVAL,
         MTEB_code,
         MTEB_multilingual_v2,
     )
@@ -63,6 +64,7 @@ def _get_previous_benchmark_names() -> dict[str, str]:
         "MTEB(Chinese)": C_MTEB.name,
         "FaMTEB(fas, beta)": FA_MTEB.name,
         "BRIGHT(long)": BRIGHT_LONG.name,
+        "VisualDocumentRetrieval": VISUAL_DOCUMENT_RETRIEVAL.name,
     }
     return previous_benchmark_names
 

mteb/leaderboard/benchmark_selector.py

@@ -110,10 +110,11 @@ class MenuEntry:
             MenuEntry(
                 "Image",
                 description=None,
-                open=False,
+                open=True,
                 benchmarks=[
-                    mteb.get_benchmark("VisualDocumentRetrieval"),
+                    mteb.get_benchmark("ViDoRe(v3)"),
                     mteb.get_benchmark("JinaVDR"),
+                    MenuEntry("Other", [mteb.get_benchmark("ViDoRe(v1&v2)")]),
                 ],
             ),
             MenuEntry(