RAI simulator support for context in QA eval (#34617)

* Changed the parameter from max_count to limit

* Add a method to have output in qa format from simulator

* simulation_result_limit was missing from main

* Return the right result variable for full_result

* to json lines supports saving context to forward it to eval

* Forward the context when context is available in full response

* Removing metadata, ch_template_placeholders from chat_protocol

* Update _utils.py

Nit fix

Author: nagkumar91

sdk/ai/azure-ai-generative/azure/ai/generative/synthetic/simulator/simulator/_callback_conversation_bot.py

@@ -45,7 +45,7 @@ async def generate_response(
 
         self.logger.info("Parsed callback response")
 
-        return response, {}, time_taken, response
+        return response, {}, time_taken, result
 
     # pylint: disable=unused-argument
     def _to_chat_protocol(self, template, conversation_history, template_parameters):

sdk/ai/azure-ai-generative/azure/ai/generative/synthetic/simulator/simulator/_utils.py

@@ -1,26 +1,65 @@
 # ---------------------------------------------------------
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
+# pylint: disable=C0303
+"""
+This module contains a utility class for managing a list of JSON lines.
+"""
 import json
 
 
 class JsonLineList(list):
+    """
+    A util to manage a list of JSON lines.
+    """
     def to_json_lines(self):
+        """
+        Converts the list to a string of JSON lines.
+        Each item in the list is converted to a JSON string 
+        and appended to the result string with a newline.
+        
+        :returns: A string of JSON lines, where each line is a JSON representation of an item in the list.
+        :rtype: str
+        """
         json_lines = ""
         for item in self:
             json_lines += json.dumps(item) + "\n"
         return json_lines
 
     def to_eval_qa_json_lines(self):
+        """
+        Converts the list to a string of JSON lines suitable for evaluation in a Q&A format.
+        Each item in the list is expected to be a dictionary with 
+        'messages' key. The 'messages' value is a list of
+        dictionaries, each with a 'role' key and a 'content' key. 
+        The 'role' value should be either 'user' or 'assistant',
+        and the 'content' value should be a string. 
+        If a 'context' key is present in the message, its value is also included
+        in the output.
+        
+        :returns: A string of JSON lines.
+        :rtype: str
+        """
         json_lines = ""
         for item in self:
             user_message = None
             assistant_message = None
-            for message in item["messages"]:
-                if message["role"] == "user":
-                    user_message = message["content"]
-                elif message["role"] == "assistant":
-                    assistant_message = message["content"]
+            context = None
+            for message in item['messages']:
+                if message['role'] == 'user':
+                    user_message = message['content']
+                elif message['role'] == 'assistant':
+                    assistant_message = message['content']
+                if 'context' in message:
+                    context = message.get("context", None)
             if user_message and assistant_message:
-                json_lines += json.dumps({"question": user_message, "answer": assistant_message}) + "\n"
+                if context:
+                    json_lines += json.dumps({
+                        'question': user_message, 
+                        'answer': assistant_message, 
+                        'context': context}) + "\n"
+                else:
+                    json_lines += json.dumps({
+                        'question': user_message, 
+                        'answer': assistant_message}) + "\n"
         return json_lines

sdk/ai/azure-ai-generative/azure/ai/generative/synthetic/simulator/simulator/simulator.py

@@ -392,13 +392,18 @@ def _get_callback_citations(self, callback_citations: dict, turn_num: Optional[i
 
     def _to_chat_protocol(self, template, conversation_history, template_parameters):
         messages = []
-
         for i, m in enumerate(conversation_history):
             message = {"content": m.message, "role": m.role.value}
             if len(template.context_key) > 0:
                 citations = self._get_citations(template_parameters, template.context_key, i)
                 message["context"] = citations
+            elif "context" in m.full_response:
+                # adding context for adv_qa
+                message["context"] = m.full_response["context"]
             messages.append(message)
+        template_parameters['metadata'] = {}
+        if "ch_template_placeholder" in template_parameters:
+            del template_parameters["ch_template_placeholder"]
 
         return {
             "template_parameters": template_parameters,