Fix-Sample-Code-Agents - Github Issue 44431 (#44441)

* sample-fix

* revert

* model fix

* Fix

* fix

Author: w-javed

sdk/ai/azure-ai-projects/.env.template

@@ -17,7 +17,7 @@
 #  `https://<your-ai-services-account-name>.services.ai.azure.com/api/projects/<your-project-name>`
 AZURE_AI_PROJECT_ENDPOINT=
 AZURE_AI_MODEL_DEPLOYMENT_NAME=
-AGENT_NAME=
+AZURE_AI_AGENT_NAME=
 CONVERSATION_ID=
 CONNECTION_NAME=
 AZURE_AI_PROJECTS_AZURE_SUBSCRIPTION_ID=

sdk/ai/azure-ai-projects/README.md

@@ -632,7 +632,7 @@ with (
     agent = project_client.agents.create_version(
         agent_name=os.environ["AZURE_AI_AGENT_NAME"],
         definition=PromptAgentDefinition(
-            model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+            model=model_deployment_name,
             instructions="You are a helpful assistant that answers general questions",
         ),
     )
@@ -643,13 +643,30 @@ with (
         item_schema={"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]},
         include_sample_schema=True,
     )
+    # Notes: for data_mapping:
+    # sample.output_text is the string output of the agent
+    # sample.output_items is the structured JSON output of the agent, including tool calls information
     testing_criteria = [
         {
             "type": "azure_ai_evaluator",
             "name": "violence_detection",
             "evaluator_name": "builtin.violence",
-            "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
-        }
+            "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"},
+        },
+        {
+            "type": "azure_ai_evaluator",
+            "name": "fluency",
+            "evaluator_name": "builtin.fluency",
+            "initialization_parameters": {"deployment_name": f"{model_deployment_name}"},
+            "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"},
+        },
+        {
+            "type": "azure_ai_evaluator",
+            "name": "task_adherence",
+            "evaluator_name": "builtin.task_adherence",
+            "initialization_parameters": {"deployment_name": f"{model_deployment_name}"},
+            "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_items}}"},
+        },
     ]
     eval_object = openai_client.evals.create(
         name="Agent Evaluation",

sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic.py

@@ -23,7 +23,7 @@
     Set these environment variables with your own values:
     1) AZURE_AI_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview
        page of your Microsoft Foundry portal.
-    2) AGENT_NAME - The name of an existing Agent in your Microsoft Foundry project.
+    2) AZURE_AI_AGENT_NAME - The name of an existing Agent in your Microsoft Foundry project.
     3) CONVERSATION_ID - The ID of an existing Conversation associated with the Agent
 """
 
@@ -34,7 +34,7 @@
 
 load_dotenv()
 
-agent_name = os.environ["AGENT_NAME"]
+agent_name = os.environ["AZURE_AI_AGENT_NAME"]
 conversation_id = os.environ["CONVERSATION_ID"]
 
 endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]

sdk/ai/azure-ai-projects/samples/agents/sample_agent_retrieve_basic_async.py

@@ -23,7 +23,7 @@
     Set these environment variables with your own values:
     1) AZURE_AI_PROJECT_ENDPOINT - The Azure AI Project endpoint, as found in the Overview
        page of your Microsoft Foundry portal.
-    2) AGENT_NAME - The name of an existing Agent in your Microsoft Foundry project.
+    2) AZURE_AI_AGENT_NAME - The name of an existing Agent in your Microsoft Foundry project.
     3) CONVERSATION_ID - The ID of an existing Conversation associated with the Agent
 """
 
@@ -36,7 +36,7 @@
 load_dotenv()
 
 endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]
-agent_name = os.environ["AGENT_NAME"]
+agent_name = os.environ["AZURE_AI_AGENT_NAME"]
 conversation_id = os.environ["CONVERSATION_ID"]
 
 

sdk/ai/azure-ai-projects/samples/evaluations/sample_agent_evaluation.py

@@ -40,6 +40,7 @@
 
 load_dotenv()
 endpoint = os.environ["AZURE_AI_PROJECT_ENDPOINT"]
+model_deployment_name = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "")  # Sample : gpt-4o-mini
 
 # [START agent_evaluation_basic]
 with (
@@ -50,7 +51,7 @@
     agent = project_client.agents.create_version(
         agent_name=os.environ["AZURE_AI_AGENT_NAME"],
         definition=PromptAgentDefinition(
-            model=os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"],
+            model=model_deployment_name,
             instructions="You are a helpful assistant that answers general questions",
         ),
     )
@@ -61,13 +62,30 @@
         item_schema={"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]},
         include_sample_schema=True,
     )
+    # Notes: for data_mapping:
+    # sample.output_text is the string output of the agent
+    # sample.output_items is the structured JSON output of the agent, including tool calls information
     testing_criteria = [
         {
             "type": "azure_ai_evaluator",
             "name": "violence_detection",
             "evaluator_name": "builtin.violence",
-            "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
-        }
+            "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"},
+        },
+        {
+            "type": "azure_ai_evaluator",
+            "name": "fluency",
+            "evaluator_name": "builtin.fluency",
+            "initialization_parameters": {"deployment_name": f"{model_deployment_name}"},
+            "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"},
+        },
+        {
+            "type": "azure_ai_evaluator",
+            "name": "task_adherence",
+            "evaluator_name": "builtin.task_adherence",
+            "initialization_parameters": {"deployment_name": f"{model_deployment_name}"},
+            "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_items}}"},
+        },
     ]
     eval_object = openai_client.evals.create(
         name="Agent Evaluation",

sdk/ai/azure-ai-projects/samples/evaluations/sample_model_evaluation.py

@@ -51,16 +51,18 @@
         item_schema={"type": "object", "properties": {"query": {"type": "string"}}, "required": ["query"]},
         include_sample_schema=True,
     )
+    # Notes: for data_mapping:
+    # {{sample.output_text}} is the string output of the provide model target for the given input in {{item.query}}
     testing_criteria = [
         {
             "type": "azure_ai_evaluator",
             "name": "violence_detection",
             "evaluator_name": "builtin.violence",
-            "data_mapping": {"query": "{{item.query}}", "response": "{{item.response}}"},
+            "data_mapping": {"query": "{{item.query}}", "response": "{{sample.output_text}}"},
         }
     ]
     eval_object = openai_client.evals.create(
-        name="Agent Evaluation",
+        name="Model Evaluation",
         data_source_config=data_source_config,
         testing_criteria=testing_criteria,  # type: ignore
     )