Bugfix for query/response based simulator (Azure#37575)

* Update prompty

* use a non preview version of API

* Remove odb

* Remove prints

* Update tests and fix the type conversion of response

* Reformmated with tox run -e black -c ../../../eng/tox/tox.ini -- .

* Update test

---------

Co-authored-by: Nagkumar Arkalgud <[email protected]>

Author: nagkumar91

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_prompty/task_query_response.prompty

@@ -33,7 +33,8 @@ Answer must not be more than 5 words
 Answer must be picked from Text as is
 Question should be as descriptive as possible and must include as much context as possible from Text
 Output must always have the provided number of QnAs
-Output must be in JSON format
+Output must be in JSON format.
+Output must have {{num_queries}} objects in the format specified below. Any other count is unacceptable.
 Text:
 <|text_start|>
 On January 24, 1984, former Apple CEO Steve Jobs introduced the first Macintosh. In late 2003, Apple had 2.06 percent of the desktop share in the United States.

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/simulator/_simulator.py

@@ -41,7 +41,7 @@ def __init__(self, azure_ai_project: Dict[str, Any], credential: Optional[Any] =
         """
         self._validate_project_config(azure_ai_project)
         self.azure_ai_project = azure_ai_project
-        self.azure_ai_project["api_version"] = "2024-02-15-preview"
+        self.azure_ai_project["api_version"] = "2024-06-01"
         self.credential = credential
 
     @staticmethod
@@ -129,7 +129,6 @@ async def __call__(
         max_conversation_turns *= 2  # account for both user and assistant turns
 
         prompty_model_config = self._build_prompty_model_config()
-
         if conversation_turns:
             return await self._simulate_with_predefined_turns(
                 target=target,
@@ -234,8 +233,16 @@ async def _simulate_with_predefined_turns(
                     target=target,
                     progress_bar=progress_bar,
                 )
-
-            simulated_conversations.append(current_simulation.to_list())
+            simulated_conversations.append(
+                JsonLineChatProtocol(
+                    {
+                        "messages": current_simulation.to_list(),
+                        "finish_reason": ["stop"],
+                        "context": {},
+                        "$schema": "http://azureml/sdk-2-0/ChatConversation.json",
+                    }
+                )
+            )
 
         progress_bar.close()
         return simulated_conversations
@@ -398,7 +405,6 @@ async def _generate_query_responses(
             prompty_model_config=prompty_model_config,
             query_response_generating_prompty_kwargs=query_response_generating_prompty_kwargs,
         )
-
         try:
             query_responses = query_flow(text=text, num_queries=num_queries)
             if isinstance(query_responses, dict):

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_non_adv_simulator.py

@@ -40,7 +40,7 @@ class TestNonAdvSimulator:
     def test_init_valid_project(self, valid_project):
         simulator = Simulator(azure_ai_project=valid_project)
         assert simulator.azure_ai_project["subscription_id"] == "test_subscription"
-        assert simulator.azure_ai_project["api_version"] == "2024-02-15-preview"
+        assert simulator.azure_ai_project["api_version"] == "2024-06-01"
 
     def test_init_invalid_project(self, invalid_project):
         with pytest.raises(ValueError):
@@ -91,7 +91,7 @@ async def test_simulate_with_predefined_turns(
         )
 
         assert len(result) == 1
-        assert isinstance(result[0], list)
+        assert isinstance(result[0], JsonLineChatProtocol)
 
     @pytest.mark.asyncio
     @patch("azure.ai.evaluation.simulator.Simulator._complete_conversation", new_callable=AsyncMock)

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_simulator.py

@@ -58,7 +58,7 @@ def test_initialization_with_all_valid_scenarios(
         for scenario in available_scenarios:
             simulator = AdversarialSimulator(azure_ai_project=azure_ai_project)
             assert callable(simulator)
-            simulator(scenario=scenario, max_conversation_turns=1, max_simulation_results=3, target=async_callback)
+            # simulator(scenario=scenario, max_conversation_turns=1, max_simulation_results=3, target=async_callback)
 
     @patch("azure.ai.evaluation.simulator._model_tools._rai_client.RAIClient._get_service_discovery_url")
     @patch(
@@ -121,4 +121,4 @@ def test_initialization_parity_with_evals(
         for scenario in available_scenarios:
             simulator = AdversarialSimulator(azure_ai_project=azure_ai_project, credential="test_credential")
             assert callable(simulator)
-            simulator(scenario=scenario, max_conversation_turns=1, max_simulation_results=3, target=async_callback)
+            # simulator(scenario=scenario, max_conversation_turns=1, max_simulation_results=3, target=async_callback)