add additional_messages parameters to forward OpenAi style messages to inference providers

src/lighteval/tasks/prompt_manager.py

@@ -107,6 +107,10 @@ def _prepare_chat_template(self, doc: Doc, tokenize: bool = True) -> str:
             messages.append({"role": "user", "content": query})
             messages.append({"role": "assistant", "content": fewshot_sample.get_golds()[0]})
 
+        # If there are any additional messages to include, lets do it
+        if doc.additional_messages:
+            messages += doc.additional_messages
+
         # Add main query
         main_query = self._extract_query(doc.query, doc.instruction)
 

src/lighteval/tasks/requests.py

@@ -202,11 +202,12 @@ class Doc:
     images: list["Image"] | None = None  # for multimodal benchmarks
     specific: dict | None = None  # Information which is specific to the current eval
 
-    # Uncoditioned query is used for PMI normalization, that's
+    # Unconditioned query is used for PMI normalization, that's
     # log P(choice | Query) - log P(choice | Unconditioned Query)
-    # The uncoditioned query shouldn't contain any information about the task, thus usually it's empty string or 'Answer:'.
+    # The unconditioned query shouldn't contain any information about the task, thus usually it's empty string or 'Answer:'.
     unconditioned_query: str | None = None
     original_query: str | None = None  # the query before preprocessing, if stored
+    additional_messages: list[object] | None = None
 
     id: str = ""
     task_name: str = ""