Fix 341 (#346)

* split greedy and sampling generative + remove small old helm mechanism
* add do_sample to generative tas criteria
* Quick fix vllm (#361)
* fix max len management in vllm
* fixed the maj@n qem being run on the same samples. needed to manage the sort and split
* add temperature to vllm config

---------

Co-authored-by: Nathan Habib <[email protected]>
Co-authored-by: Nathan Habib <[email protected]>

Author: clefourrier

src/lighteval/data.py

@@ -264,7 +264,7 @@ def init_split_limits(self, num_dataset_splits):
         splits_indices = [tuple(e) for e in splits_indices]
         return num_dataset_splits, splits_indices
 
-    def _sorting_criteria(self, request: GreedyUntilRequest) -> tuple[bool, list, int]:
+    def _sorting_criteria(self, request: GreedyUntilRequest) -> tuple[bool, bool, list, int]:
         """
         Collate function for generating batches.
 
@@ -279,7 +279,7 @@ def _sorting_criteria(self, request: GreedyUntilRequest) -> tuple[bool, list, in
         # The generative task has no limit except the model context
         if gen_length is None:
             gen_length = 0
-        return request.use_logits, request.stop_sequence, -(len(toks) + gen_length)
+        return request.do_sample, request.use_logits, request.stop_sequence, -(len(toks) + gen_length)
 
 
 class GenerativeTaskDatasetNanotron(GenerativeTaskDataset):

src/lighteval/metrics/__init__.py

@@ -90,14 +90,21 @@ def apply_generative_metric(  # noqa: C901
     formatted_docs: list[Doc],
     metrics: list[Metric],
     output_regex: str = None,
-    max_num_samples: int = 1,
 ):
     outputs = []
 
     for sample_id, results, formatted_doc in zip(sample_ids, responses, formatted_docs):
         output = {}
 
+        # Extracting gold
+        try:
+            golds = formatted_doc.get_golds()
+        except (KeyError, IndexError):
+            golds = None
+
+        # Post processing prediction
         if len(results) > 1:
+            # In case of sampling, it's a list of one list of n samples
             raise Exception("You returned more than one result for a sample with a generative metric.")
         results = results[0]
 
@@ -112,38 +119,14 @@ def apply_generative_metric(  # noqa: C901
                 pred = pred_raw
             preds.append(pred)
 
-        # Extracting gold
-        try:
-            golds = formatted_doc.get_golds()
-        except (KeyError, IndexError):
-            golds = None
-
-        # Specific process for HELM like evals # hrm
-        # if "label_to_choices" in formatted_doc:
-        if formatted_doc.specific is not None and "label_to_choices" in formatted_doc.specific:
-            # Helm predicts on labels keys (A/B/C/D), but computes metrics on choices
-            preds = [formatted_doc.specific["label_to_choices"].get(p) for p in preds]
-            golds = [formatted_doc.specific["label_to_choices"][g] for g in golds]
-
         for metric in metrics:
-            if metric.category == MetricCategory.GENERATIVE:
-                output.update(
-                    metric.compute(
-                        golds=golds,
-                        predictions=as_list(preds[0]) if max_num_samples > 1 else preds,
-                        formatted_doc=formatted_doc,
-                    )
+            output.update(
+                metric.compute(
+                    golds=golds,
+                    predictions=preds,
+                    formatted_doc=formatted_doc,
                 )
-            if metric.category == MetricCategory.GENERATIVE_LOGPROB:
-                output.update(
-                    metric.compute(
-                        golds=golds,
-                        predictions=as_list(preds[0]) if max_num_samples > 1 else preds,
-                        formatted_doc=formatted_doc,
-                    )
-                )
-            if metric.category == MetricCategory.GENERATIVE_SAMPLING:
-                output.update(metric.compute(golds=golds, predictions=preds, formatted_doc=formatted_doc))
+            )
         outputs.append(output)
 
     return outputs

src/lighteval/models/base_model.py

@@ -537,6 +537,7 @@ def greedy_until(
                 max_new_tokens = batch[0].generation_size
                 returns_logits = batch[0].use_logits
                 num_samples = batch[0].num_samples
+                do_sample = batch[0].do_sample
 
                 context = [c.context for c in batch]
 
@@ -590,6 +591,7 @@ def greedy_until(
                     stop_tokens=stop_tokens,
                     returns_logits=returns_logits,
                     num_samples=num_samples,
+                    do_sample=do_sample,
                 )
                 results.extend(cur_reponses)
 
@@ -602,6 +604,7 @@ def _generate(
         stop_tokens: list[str],
         returns_logits: Optional[bool] = False,
         num_samples: Optional[int] = 1,
+        do_sample: Optional[bool] = False,
     ) -> list[GenerativeResponse]:
         """Contains the actual logic of the generation.
         First computes the stop sequences, then generates the predictions, then converts the outputs to GenerativeResponse.
@@ -619,7 +622,7 @@ def _generate(
             return_dict_in_generate=True,
             output_scores=True,
             eos_token_id=self.tokenizer.eos_token_id,
-            do_sample=num_samples > 1,
+            do_sample=do_sample,
             num_return_sequences=num_samples,
         )
         if returns_logits:
@@ -660,10 +663,6 @@ def _generate(
 
                 decoded_generations.append(decoded_generation)
 
-            if num_samples == 1:  # We only return one item
-                result_generations = result_generations[0]
-                decoded_generations = decoded_generations[0]
-
             cur_response = GenerativeResponse(
                 result=decoded_generations,
                 logits=logits[ix][: len_logits[ix]] if returns_logits else None,

src/lighteval/models/model_config.py

@@ -229,6 +229,7 @@ class VLLMModelConfig:
     pairwise_tokenization: bool = False  # whether to tokenize the context and continuation separately or together.
 
     subfolder: Optional[str] = None
+    temperature: float = 0.6  # will be used for multi sampling tasks, for tasks requiring no sampling, this will be ignored and set to 0.
 
 
 @dataclass

src/lighteval/models/vllm_model.py

@@ -77,10 +77,7 @@ def __init__(
         self._add_special_tokens = config.add_special_tokens if config.add_special_tokens is not None else False
         self._tokenizer = self._create_auto_tokenizer(config, env_config)
 
-        if config.max_model_length is not None:
-            self._max_length = int(config.max_model_length)
-        else:
-            self._max_length = self.tokenizer.model_max_length or self.tokenizer.max_position_embeddings
+        self._max_length = int(config.max_model_length) if config.max_model_length is not None else None
 
         # If model_parallel is not set we compare the number of processes with the number of GPUs
         self.model = self._create_auto_model(config, env_config)
@@ -152,6 +149,13 @@ def _create_auto_model(self, config: VLLMModelConfig, env_config: EnvConfig) ->
             return None
 
         model = LLM(**self.model_args)
+
+        # If the max_length can't get extracted from the config, it will be inferred from the model
+        # Inferring from the tokenizer will cause vllm to bug for models with mismatches between model
+        # config and tk config, like mistralai/Mistral-7B-v0.1
+        if self._max_length is None:
+            self._max_length = model.llm_engine.model_config.max_seq_len_to_capture
+
         return model
 
     def _create_auto_tokenizer(self, config: VLLMModelConfig, env_config: EnvConfig):
@@ -164,36 +168,6 @@ def _create_auto_tokenizer(self, config: VLLMModelConfig, env_config: EnvConfig)
         tokenizer.pad_token = tokenizer.eos_token
         return tokenizer
 
-    def _init_max_length(self, max_length) -> int:
-        """Return the maximum sequence length of the model.
-        NOTE: Different model configurations have different max sequence length
-        attribute names.
-            - n_positions: (CTRLConfig)
-            - max_position_embeddings: (BartConfig, RoFormerConfig)
-            - n_ctx: (GPT2Config)
-        NOTE: For relative position encoded models you should specify the max
-        sequence length of the model in the constructor via `max_length`.
-
-        Args:
-            max_length (Optional[int]): The maximum length of the input sequence. If not provided, it will be determined
-                based on the model's configuration or tokenizer's model_max_length attribute.
-
-        Returns:
-            int: Max length to use depending on the available args and config
-        """
-        if max_length is not None:
-            return int(max_length)
-        # Try to get the sequence length from the model config.
-        seqlen_config_attrs = ("n_positions", "max_position_embeddings", "n_ctx")
-
-        for attr in seqlen_config_attrs:
-            if hasattr(self._config, attr):
-                return getattr(self._config, attr)
-
-        # Default max sequence length setting for when no `max_length` is provided
-        # or no max length config setting is found in the model or tokenizer.
-        return 2048
-
     def greedy_until(
         self,
         requests: list[GreedyUntilRequest],
@@ -300,7 +274,7 @@ def _generate(
         """Contains the actual logic of the generation."""
         if generate:
             sampling_params = SamplingParams(
-                temperature=1.0 if num_samples > 1 else 0.0,
+                temperature=float(self._config.temperature) if num_samples > 1 else 0.0,
                 n=num_samples,
                 max_tokens=max_new_tokens,
                 stop=stop_tokens,

src/lighteval/pipeline.py

@@ -302,7 +302,10 @@ def _compute_metrics(self, sample_id_to_responses):
                 metric_category_metrics = [metric for metric in task.metrics if metric.category == metric_category]
 
                 outputs = metric_function(
-                    sample_ids=sample_ids, responses=responses, formatted_docs=docs, metrics=metric_category_metrics
+                    sample_ids=sample_ids,
+                    responses=responses,
+                    formatted_docs=docs,
+                    metrics=metric_category_metrics,
                 )
 
                 for output, doc, response in zip(outputs, docs, responses):

src/lighteval/tasks/default_prompts.py

@@ -786,7 +786,7 @@ def hellaswag_harness(line, task_name: str = None):
     )
 
 
-def hellaswag_helm(line, task_name: str = None):
+def hellaswag_generative(line, task_name: str = None):
     query = "The following are multiple choice questions (with answers) about common sense.\n\n"
     query += f"Question: {line['activity_label']}: {line['ctx_a']} {line['ctx_b'].capitalize()}\n"
     query += "".join([f"{key}. {choice}\n" for key, choice in zip(LETTER_INDICES, line["endings"])])
@@ -800,9 +800,6 @@ def hellaswag_helm(line, task_name: str = None):
         gold_index=gold_ix,  # -1 for test,
         instruction="The following are multiple choice questions (with answers) about common sense.\n\n",
         target_for_fewshot_sorting=line["endings"][gold_ix] if gold_ix > -1 else "",
-        specific={
-            "label_to_choices": {f" {key}": choice for key, choice in zip(LETTER_INDICES, line["endings"])},
-        },
     )
 
 

src/lighteval/tasks/default_tasks.py

@@ -8708,10 +8708,10 @@
     trust_dataset=True,
     version=0,
 )
-hellaswag_helm = LightevalTaskConfig(
+hellaswag_generative = LightevalTaskConfig(
     name="hellaswag",
     suite=["helm", "helm_general"],
-    prompt_function=prompt.hellaswag_helm,
+    prompt_function=prompt.hellaswag_generative,
     hf_repo="hellaswag",
     hf_subset="default",
     hf_avail_splits=["train", "test", "validation"],

src/lighteval/tasks/lighteval_task.py

@@ -395,12 +395,29 @@ def construct_requests(
                     metric_categories=[MetricCategory.PERPLEXITY],
                 )
             ]
+        if self.has_metric_category[MetricCategory.GENERATIVE_SAMPLING]:
+            # All the possible sampling tasks require the same generation process - we can do them in one step
+            # so we select the maximum number of samples and the metrics will select only the
+            # relevant number of tiems
+            requests[RequestType.GREEDY_UNTIL] += [
+                GreedyUntilRequest(
+                    task_name=current_task_name,
+                    sample_index=document_id_seed,
+                    request_index=0,
+                    context=context,
+                    stop_sequence=self.stop_sequence,
+                    generation_size=self.generation_size,
+                    generation_grammar=self.generation_grammar,
+                    num_samples=max(self.num_samples),
+                    do_sample=True,
+                    use_logits=False,
+                    metric_categories=[MetricCategory.GENERATIVE_SAMPLING],
+                )
+            ]
         if (
-            self.has_metric_category[MetricCategory.GENERATIVE_SAMPLING]
-            or self.has_metric_category[MetricCategory.GENERATIVE]
+            self.has_metric_category[MetricCategory.GENERATIVE]
             or self.has_metric_category[MetricCategory.GENERATIVE_LOGPROB]
         ):
-            # All these tasks require the same generation process - we can do them in one step
             use_logits = self.has_metric_category[MetricCategory.GENERATIVE_LOGPROB]
             requests[RequestType.GREEDY_UNTIL] += [
                 GreedyUntilRequest(
@@ -411,12 +428,11 @@ def construct_requests(
                     stop_sequence=self.stop_sequence,
                     generation_size=self.generation_size,
                     generation_grammar=self.generation_grammar,
-                    num_samples=max(self.num_samples),  # If we have several samplings to apply, we use the max
+                    num_samples=1,
                     use_logits=use_logits,
                     metric_categories=[
                         c
                         for c in [
-                            MetricCategory.GENERATIVE_SAMPLING,
                             MetricCategory.GENERATIVE,
                             MetricCategory.GENERATIVE_LOGPROB,
                         ]
@@ -443,7 +459,6 @@ def construct_requests(
                 )
                 for i, choice in enumerate(formatted_doc.choices)
             ]
-
         if self.has_metric_category[MetricCategory.MULTICHOICE_PMI]:
             assert (
                 formatted_doc.unconditioned_query is not None

src/lighteval/tasks/requests.py

@@ -126,6 +126,7 @@ class GreedyUntilRequest(Request):
     request_type = RequestType.GREEDY_UNTIL
     tokenized_context: list[int] = None
     num_samples: int = None
+    do_sample: bool = False
     use_logits: bool = False