Adds RULER benchmark

src/lighteval/metrics/metrics.py

@@ -133,7 +133,27 @@ class Metrics(Enum):
         corpus_level_fn=np.mean,
         higher_is_better=True,
     )
-
+    ruler_match_any = SampleLevelMetric(
+        metric_name="ruler_match_any",
+        sample_level_fn=lambda predictions, golds, formatted_doc: max(
+            [1.0 if r.lower() in predictions[0].lower() else 0.0 for r in golds]
+        ),
+        category=MetricCategory.GENERATIVE,
+        use_case=MetricUseCase.SUMMARIZATION,
+        corpus_level_fn=np.mean,
+        higher_is_better=True,
+    )
+    ruler_match_all = SampleLevelMetric(
+        metric_name="ruler_match_all",
+        sample_level_fn=lambda predictions, golds, formatted_doc: sum(
+            [1.0 if r.lower() in predictions[0].lower() else 0.0 for r in golds]
+        )
+        / len(golds),
+        category=MetricCategory.GENERATIVE,
+        use_case=MetricUseCase.SUMMARIZATION,
+        corpus_level_fn=np.mean,
+        higher_is_better=True,
+    )
     bleurt = SampleLevelMetric(
         metric_name="bleurt",
         sample_level_fn=BLEURT().compute,

src/lighteval/models/vllm/vllm_model.py

@@ -266,8 +266,9 @@ def greedy_until(
             if max_new_tokens is not None:
                 if context_size + max_new_tokens > self.max_length:
                     logger.warning(
-                        f"{context_size + max_new_tokens=} which is greater than {self.max_length=}. Truncating context to {self.max_length - max_new_tokens} tokens."
+                        f"{context_size + max_new_tokens=} which is greater than {self.max_length=}. Truncating context to {self.max_length=} - {max_new_tokens=} = {self.max_length - max_new_tokens} tokens."
                     )
+                    breakpoint()
                     context_size = self.max_length - max_new_tokens
                     if context_size < 0:
                         logger.critical(

src/lighteval/tasks/default_prompts.py

@@ -44,6 +44,15 @@
 # fmt: on
 
 
+def ruler(line, task_name: str = None):
+    query = line["input"]
+    choices = line["outputs"]
+    gold_index = 0
+    instruction = "Only answer the question to complete the prompt, without any additional text.\n"
+    query = f"{instruction}{query}"
+
+    return Doc(query=query, instruction=instruction, choices=choices, gold_index=gold_index, task_name=task_name)
+
 def mmmu_pro(line, task_name: Optional[str] = None):
     # fmt: off
     question = line["question"]        # "What is the capital of France?"
@@ -87,7 +96,6 @@ def mmmu_pro(line, task_name: Optional[str] = None):
         instruction=instructions,
     )
 
-
 def mmmu_pro_vision(line, task_name: str = None):
     instruction = (
         "Answer with the option letter from the given choices directly."
@@ -119,14 +127,17 @@ def mmmu_pro_vision(line, task_name: str = None):
         instruction=instruction,
     )
 
-
 def simpleqa(line, task_name: str = None):
     query = line["problem"]
     choices = [line["answer"]]
     gold_index = 0
 
     return Doc(
-        task_name=task_name, query=query, choices=choices, gold_index=gold_index, specific={**eval(line["metadata"])}
+        task_name=task_name,
+        query=query,
+        choices=choices,
+        gold_index=gold_index,
+        specific={**eval(line["metadata"])},
     )
 
 
@@ -254,7 +265,7 @@ def arc_with_options(line, task_name: str = None):
     query += "".join([f"\n{key}. {choice}" for key, choice in zip(LETTER_INDICES, line["choices"]["text"])])
     query += "\nAnswer:"
     return Doc(
-        task_name=task_name,
+mm        task_name=task_name,
         query=query,
         choices=line["choices"]["text"],
         gold_index=line["choices"]["label"].index(line["answerKey"]),