Skip to content

Add Samples for Agentic Evaluators in azure-ai-projects #43766

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
wants to merge 6 commits into from
Closed

Add Samples for Agentic Evaluators in azure-ai-projects #43766

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
6910554
Add Samples for Agentic Evaluators in azure-ai-projects
m7md7sien Nov 3, 2025
b93a835
Add Samples for Agentic Evaluators in azure-ai-projects
m7md7sien Nov 3, 2025
8513df4
merge
m7md7sien Nov 3, 2025
faa7eec
Add fluency samples
m7md7sien Nov 3, 2025
88959d0
address comments
m7md7sien Nov 3, 2025
2b02afd
update dependencies
m7md7sien Nov 3, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
188 changes: 188 additions & 0 deletions sdk/ai/azure-ai-projects/samples/evaluation/sample_agentic_evaluators/sample_coherence.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,188 @@
# pylint: disable=line-too-long,useless-suppression
# ------------------------------------
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# ------------------------------------

"""
DESCRIPTION:
Given an AIProjectClient, this sample demonstrates how to use the synchronous
`openai.evals.*` methods to create, get and list eval group and and eval runs
using inline dataset content.

USAGE:
python sample_coherence.py

Before running the sample:

pip install "azure-ai-projects>=2.0.0b1" azure-identity python-dotenv

Set these environment variables with your own values:
1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your
Azure AI Foundry project. It has the form: https://<account_name>.services.ai.azure.com/api/projects/<project_name>.
2) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation.
"""

from dotenv import load_dotenv
import json
import os
import time

from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from openai.types.evals.create_eval_jsonl_run_data_source_param import (
CreateEvalJSONLRunDataSourceParam,
SourceFileContent,
SourceFileContentContent
)


load_dotenv()


def main() -> None:
endpoint = os.environ[
"AZURE_AI_PROJECT_ENDPOINT"
] # Sample : https://<account_name>.services.ai.azure.com/api/projects/<project_name>
model_deployment_name = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") # Sample : gpt-4o-mini

with DefaultAzureCredential() as credential:
with AIProjectClient(endpoint=endpoint, credential=credential, api_version="2025-11-15-preview") as project_client:
print("Creating an OpenAI client from the AI Project client")

client = project_client.get_openai_client()
client._custom_query = {"api-version": "2025-11-15-preview"}

data_source_config = {
"type": "custom",
"item_schema": {
"type": "object",
"properties": {
"query": {
"type": "string"
},
"response": {
"type": "string"
}
},
"required": []
},
"include_sample_schema": True
}

testing_criteria = [
{
"type": "azure_ai_evaluator",
"name": "coherence",
"evaluator_name": "builtin.coherence",
"initialization_parameters": {
"deployment_name": f"{model_deployment_name}"
},
"data_mapping": {
"query": "{{item.query}}",
"response": "{{item.response}}"
}
}
]

print("Creating Eval Group")
eval_object = client.evals.create(
name="Test Coherence Evaluator with inline data",
data_source_config=data_source_config,
testing_criteria=testing_criteria,
)
print(f"Eval Group created")

print("Get Eval Group by Id")
eval_object_response = client.evals.retrieve(eval_object.id)
print("Eval Run Response:")
pprint(eval_object_response)

# Sample inline data
success_query = "What is the capital of France?"
success_response = "The capital of France is Paris."

# Failure example - incoherent response
failure_query = "What is the capital of France?"
failure_response = "France capital is... well, the city where government sits is Paris but no wait, Lyon is bigger actually maybe Rome? The French people live in many cities but the main one, I think it's definitely Paris or maybe not, depends on what you mean by capital."

print("Creating Eval Run with Inline Data")
eval_run_object = client.evals.runs.create(
eval_id=eval_object.id,
name="inline_data_run",
metadata={
"team": "eval-exp",
"scenario": "inline-data-v1"
},
data_source=CreateEvalJSONLRunDataSourceParam(
type="jsonl",
source=SourceFileContent(
type="file_content",
content= [
# Success example - coherent response
SourceFileContentContent(
item= {
"query": success_query,
"response": success_response
}
),
# Failure example - incoherent response
SourceFileContentContent(
item= {
"query": failure_query,
"response": failure_response
}
)
]
)
)
)

print(f"Eval Run created")
pprint(eval_run_object)

print("Get Eval Run by Id")
eval_run_response = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id)
print("Eval Run Response:")
pprint(eval_run_response)

print("\n\n----Eval Run Output Items----\n\n")

while True:
run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id)
if run.status == "completed" or run.status == "failed":
output_items = list(client.evals.runs.output_items.list(
run_id=run.id, eval_id=eval_object.id
))
pprint(output_items)
print(f"Eval Run Status: {run.status}")
print(f"Eval Run Report URL: {run.report_url}")
break
time.sleep(5)
print("Waiting for eval run to complete...")

# [END evaluations_sample]


def _to_json_primitive(obj):
if obj is None or isinstance(obj, (str, int, float, bool)):
return obj
if isinstance(obj, (list, tuple)):
return [_to_json_primitive(i) for i in obj]
if isinstance(obj, dict):
return {k: _to_json_primitive(v) for k, v in obj.items()}
for method in ("to_dict", "as_dict", "dict", "serialize"):
if hasattr(obj, method):
try:
return _to_json_primitive(getattr(obj, method)())
except Exception:
pass
if hasattr(obj, "__dict__"):
return _to_json_primitive({k: v for k, v in vars(obj).items() if not k.startswith("_")})
return str(obj)

def pprint(str) -> None:
print(json.dumps(_to_json_primitive(str), indent=2))

if __name__ == "__main__":
main()
176 changes: 176 additions & 0 deletions sdk/ai/azure-ai-projects/samples/evaluation/sample_agentic_evaluators/sample_fluency.py
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,176 @@
# pylint: disable=line-too-long,useless-suppression
# ------------------------------------
# Copyright (c) Microsoft Corporation.
# Licensed under the MIT License.
# ------------------------------------

"""
DESCRIPTION:
Given an AIProjectClient, this sample demonstrates how to use the synchronous
`openai.evals.*` methods to create, get and list eval group and and eval runs
using inline dataset content.

USAGE:
python sample_fluency.py

Before running the sample:

pip install "azure-ai-projects>=2.0.0b1" azure-identity python-dotenv

Set these environment variables with your own values:
1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your
Azure AI Foundry project. It has the form: https://<account_name>.services.ai.azure.com/api/projects/<project_name>.
2) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation.
"""

from dotenv import load_dotenv
import os
import json
import time

from azure.identity import DefaultAzureCredential
from azure.ai.projects import AIProjectClient
from openai.types.evals.create_eval_jsonl_run_data_source_param import (
CreateEvalJSONLRunDataSourceParam,
SourceFileContent,
SourceFileContentContent
)


load_dotenv()


def main() -> None:
endpoint = os.environ[
"AZURE_AI_PROJECT_ENDPOINT"
] # Sample : https://<account_name>.services.ai.azure.com/api/projects/<project_name>
model_deployment_name = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") # Sample : gpt-4o-mini

with DefaultAzureCredential() as credential:
with AIProjectClient(endpoint=endpoint, credential=credential, api_version="2025-11-15-preview") as project_client:
print("Creating an OpenAI client from the AI Project client")

client = project_client.get_openai_client()
client._custom_query = {"api-version": "2025-11-15-preview"}

data_source_config = {
"type": "custom",
"item_schema": {
"type": "object",
"properties": {
"query": {
"type": "string"
},
"response": {
"type": "string"
}
},
"required": []
},
"include_sample_schema": True
}

testing_criteria = [
{
"type": "azure_ai_evaluator",
"name": "coherence",
"evaluator_name": "builtin.coherence",
"initialization_parameters": {
"deployment_name": f"{model_deployment_name}"
},
"data_mapping": {
"query": "{{item.query}}",
"response": "{{item.response}}"
}
}
]

print("Creating Eval Group")
eval_object = client.evals.create(
name="Test Fluency Evaluator with inline data",
data_source_config=data_source_config,
testing_criteria=testing_criteria,
)
print(f"Eval Group created")

print("Get Eval Group by Id")
eval_object_response = client.evals.retrieve(eval_object.id)
print("Eval Run Response:")
pprint(eval_object_response)

# Sample inline data
query = "What is the capital of France?"
response = "The capital of France is Paris."

print("Creating Eval Run with Inline Data")
eval_run_object = client.evals.runs.create(
eval_id=eval_object.id,
name="inline_data_run",
metadata={
"team": "eval-exp",
"scenario": "inline-data-v1"
},
data_source=CreateEvalJSONLRunDataSourceParam(
type="jsonl",
source=SourceFileContent(
type="file_content",
content= [
SourceFileContentContent(
item= {
"query": query,
"response": response
}
)
]
)
)
)

print(f"Eval Run created")
pprint(eval_run_object)

print("Get Eval Run by Id")
eval_run_response = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id)
print("Eval Run Response:")
pprint(eval_run_response)

print("\n\n----Eval Run Output Items----\n\n")

while True:
run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id)
if run.status == "completed" or run.status == "failed":
output_items = list(client.evals.runs.output_items.list(
run_id=run.id, eval_id=eval_object.id
))
pprint(output_items)
print(f"Eval Run Status: {run.status}")
print(f"Eval Run Report URL: {run.report_url}")
break
time.sleep(5)
print("Waiting for eval run to complete...")

# [END evaluations_sample]


def _to_json_primitive(obj):
if obj is None or isinstance(obj, (str, int, float, bool)):
return obj
if isinstance(obj, (list, tuple)):
return [_to_json_primitive(i) for i in obj]
if isinstance(obj, dict):
return {k: _to_json_primitive(v) for k, v in obj.items()}
for method in ("to_dict", "as_dict", "dict", "serialize"):
if hasattr(obj, method):
try:
return _to_json_primitive(getattr(obj, method)())
except Exception:
pass
if hasattr(obj, "__dict__"):
return _to_json_primitive({k: v for k, v in vars(obj).items() if not k.startswith("_")})
return str(obj)

def pprint(str) -> None:
print(json.dumps(_to_json_primitive(str), indent=2))

if __name__ == "__main__":
main()
Loading
Loading