Skip to content

Commit 94fd5a5

Browse files
w-javedw-javed
committed
adding ai assisted
1 parent 4561ed4 commit 94fd5a5

File tree

5 files changed

+219
-6
lines changed

5 files changed

+219
-6
lines changed

‎sdk/ai/azure-ai-projects/samples/evaluation/sample_evaluations_ai_assisted.py‎

Lines changed: 213 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,213 @@
1+
# pylint: disable=line-too-long,useless-suppression
2+
# ------------------------------------
3+
# Copyright (c) Microsoft Corporation.
4+
# Licensed under the MIT License.
5+
# ------------------------------------
6+
7+
"""
8+
DESCRIPTION:
9+
Given an AIProjectClient, this sample demonstrates how to use the synchronous
10+
`openai.evals.*` methods to create, get and list eval group and and eval runs.
11+
12+
USAGE:
13+
python sample_evaluations_ai_assisted.py
14+
15+
Before running the sample:
16+
17+
pip install "azure-ai-projects>=2.0.0b1" azure-identity python-dotenv
18+
19+
Set these environment variables with your own values:
20+
1) AZURE_AI_PROJECT_ENDPOINT - Required. The Azure AI Project endpoint, as found in the overview page of your
21+
Azure AI Foundry project. It has the form: https://<account_name>.services.ai.azure.com/api/projects/<project_name>.
22+
2) CONNECTION_NAME - Required. The name of the connection of type Azure Storage Account, to use for the dataset upload.
23+
3) MODEL_ENDPOINT - Required. The Azure OpenAI endpoint associated with your Foundry project.
24+
It can be found in the Foundry overview page. It has the form https://<account_name>.openai.azure.com.
25+
4) MODEL_API_KEY - Required. The API key for the model endpoint. Can be found under "key" in the model details page
26+
(click "Models + endpoints" and select your model to get to the model details page).
27+
5) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation.
28+
6) DATASET_NAME - Optional. The name of the Dataset to create and use in this sample.
29+
7) DATASET_VERSION - Optional. The version of the Dataset to create and use in this sample.
30+
8) DATA_FOLDER - Optional. The folder path where the data files for upload are located.
31+
"""
32+
33+
import os
34+
35+
from azure.identity import DefaultAzureCredential
36+
from azure.ai.projects import AIProjectClient
37+
from azure.ai.projects.models import (
38+
DatasetVersion,
39+
)
40+
import json
41+
import time
42+
from pprint import pprint
43+
from openai.types.evals.create_eval_jsonl_run_data_source_param import CreateEvalJSONLRunDataSourceParam, SourceFileID
44+
from dotenv import load_dotenv
45+
from datetime import datetime
46+
47+
48+
load_dotenv()
49+
50+
endpoint = os.environ[
51+
"AZURE_AI_PROJECT_ENDPOINT"
52+
] # Sample : https://<account_name>.services.ai.azure.com/api/projects/<project_name>
53+
54+
connection_name = os.environ.get("CONNECTION_NAME", "")
55+
model_endpoint = os.environ.get("MODEL_ENDPOINT", "") # Sample: https://<account_name>.openai.azure.com.
56+
model_api_key = os.environ.get("MODEL_API_KEY", "")
57+
model_deployment_name = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") # Sample : gpt-4o-mini
58+
dataset_name = os.environ.get("DATASET_NAME", "")
59+
dataset_version = os.environ.get("DATASET_VERSION", "1")
60+
61+
# Construct the paths to the data folder and data file used in this sample
62+
script_dir = os.path.dirname(os.path.abspath(__file__))
63+
data_folder = os.environ.get("DATA_FOLDER", os.path.join(script_dir, "data_folder"))
64+
data_file = os.path.join(data_folder, "sample_data_evaluation.jsonl")
65+
66+
with DefaultAzureCredential() as credential:
67+
68+
with AIProjectClient(endpoint=endpoint, credential=credential) as project_client:
69+
70+
print("Upload a single file and create a new Dataset to reference the file.")
71+
dataset: DatasetVersion = project_client.datasets.upload_file(
72+
name=dataset_name or f"eval-data-{datetime.utcnow().strftime('%Y-%m-%d_%H%M%S_UTC')}",
73+
version=dataset_version,
74+
file_path=data_file,
75+
)
76+
pprint(dataset)
77+
78+
print("Creating an OpenAI client from the AI Project client")
79+
80+
client = project_client.get_openai_client()
81+
82+
data_source_config = {
83+
"type": "custom",
84+
"item_schema": {
85+
"type": "object",
86+
"properties": {
87+
"response": {"type": "string"},
88+
"ground_truth": {"type": "string"},
89+
},
90+
"required": [],
91+
},
92+
"include_sample_schema": False,
93+
}
94+
95+
testing_criteria = [
96+
{
97+
"type": "azure_ai_evaluator",
98+
"name": "Similarity",
99+
"evaluator_name": "builtin.similarity",
100+
"data_mapping": {
101+
"response": "{{item.answer}}",
102+
"ground_truth": "{{item.ground_truth}}"
103+
},
104+
"initialization_parameters": {
105+
"deployment_name": f"{model_deployment_name}",
106+
"threshold": 3
107+
}
108+
},
109+
{
110+
"type": "azure_ai_evaluator",
111+
"name": "ROUGEScore",
112+
"evaluator_name": "builtin.rouge_score",
113+
"data_mapping": {
114+
"response": "{{item.answer}}",
115+
"ground_truth": "{{item.ground_truth}}"
116+
},
117+
"initialization_parameters": {
118+
"rouge_type": "rouge1",
119+
"f1_score_threshold": 0.5,
120+
"precision_threshold": 0.5,
121+
"recall_threshold": 0.5
122+
}
123+
},
124+
{
125+
"type": "azure_ai_evaluator",
126+
"name": "METEORScore",
127+
"evaluator_name": "builtin.meteor_score",
128+
"data_mapping": {
129+
"response": "{{item.answer}}",
130+
"ground_truth": "{{item.ground_truth}}"
131+
},
132+
"initialization_parameters": {
133+
"threshold": 0.5
134+
}
135+
},
136+
{
137+
"type": "azure_ai_evaluator",
138+
"name": "GLEUScore",
139+
"evaluator_name": "builtin.gleu_score",
140+
"data_mapping": {
141+
"response": "{{item.answer}}",
142+
"ground_truth": "{{item.ground_truth}}"
143+
},
144+
"initialization_parameters": {
145+
"threshold": 0.5
146+
}
147+
},
148+
{
149+
"type": "azure_ai_evaluator",
150+
"name": "F1Score",
151+
"evaluator_name": "builtin.f1_score",
152+
"data_mapping": {
153+
"response": "{{item.answer}}",
154+
"ground_truth": "{{item.ground_truth}}"
155+
},
156+
"initialization_parameters": {
157+
"threshold": 0.5
158+
}
159+
},
160+
{
161+
"type": "azure_ai_evaluator",
162+
"name": "BLEUScore",
163+
"evaluator_name": "builtin.bleu_score",
164+
"data_mapping": {
165+
"response": "{{item.answer}}",
166+
"ground_truth": "{{item.ground_truth}}"
167+
},
168+
"initialization_parameters": {
169+
"threshold": 0.5
170+
}
171+
}
172+
]
173+
174+
print("Creating Eval Group")
175+
eval_object = client.evals.create(
176+
name="ai assisted evaluators test",
177+
data_source_config=data_source_config,
178+
testing_criteria=testing_criteria,
179+
)
180+
print(f"Eval Group created")
181+
182+
print("Get Eval Group by Id")
183+
eval_object_response = client.evals.retrieve(eval_object.id)
184+
print("Eval Run Response:")
185+
pprint(eval_object_response)
186+
187+
print("Creating Eval Run")
188+
eval_run_object = client.evals.runs.create(
189+
eval_id=eval_object.id,
190+
name="dataset",
191+
metadata={"team": "eval-exp", "scenario": "notifications-v1"},
192+
data_source=CreateEvalJSONLRunDataSourceParam(
193+
source=SourceFileID(id=dataset.id or "", type="file_id"), type="jsonl"
194+
),
195+
)
196+
print(f"Eval Run created")
197+
pprint(eval_run_object)
198+
199+
print("Get Eval Run by Id")
200+
eval_run_response = client.evals.runs.retrieve(run_id=eval_run_object.id, eval_id=eval_object.id)
201+
print("Eval Run Response:")
202+
pprint(eval_run_response)
203+
204+
while True:
205+
run = client.evals.runs.retrieve(run_id=eval_run_response.id, eval_id=eval_object.id)
206+
if run.status == "completed" or run.status == "failed":
207+
output_items = list(client.evals.runs.output_items.list(run_id=run.id, eval_id=eval_object.id))
208+
pprint(output_items)
209+
print(f"Eval Run Report URL: {run.report_url}")
210+
211+
break
212+
time.sleep(5)
213+
print("Waiting for eval run to complete...")

‎sdk/ai/azure-ai-projects/samples/evaluation/sample_evaluations_builtin_with_dataset_id.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
It can be found in the Foundry overview page. It has the form https://<account_name>.openai.azure.com.
2626
4) MODEL_API_KEY - Required. The API key for the model endpoint. Can be found under "key" in the model details page
2727
(click "Models + endpoints" and select your model to get to the model details page).
28-
5) MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation.
28+
5) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation.
2929
6) DATASET_NAME - Optional. The name of the Dataset to create and use in this sample.
3030
7) DATASET_VERSION - Optional. The version of the Dataset to create and use in this sample.
3131
8) DATA_FOLDER - Optional. The folder path where the data files for upload are located.

‎sdk/ai/azure-ai-projects/samples/evaluation/sample_evaluations_builtin_with_inline_data.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
It can be found in the Foundry overview page. It has the form https://<account_name>.openai.azure.com.
2626
4) MODEL_API_KEY - Required. The API key for the model endpoint. Can be found under "key" in the model details page
2727
(click "Models + endpoints" and select your model to get to the model details page).
28-
5) MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation.
28+
5) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation.
2929
6) DATASET_NAME - Optional. The name of the Dataset to create and use in this sample.
3030
7) DATASET_VERSION - Optional. The version of the Dataset to create and use in this sample.
3131
8) DATA_FOLDER - Optional. The folder path where the data files for upload are located.

‎sdk/ai/azure-ai-projects/samples/evaluation/sample_evaluations_builtin_with_traces.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
2) APPINSIGHTS_RESOURCE_ID - Required. The Azure Application Insights resource ID that stores agent traces.
2525
It has the form: /subscriptions/<subscription_id>/resourceGroups/<rg_name>/providers/Microsoft.Insights/components/<resource_name>.
2626
3) AGENT_ID - Required. The agent identifier emitted by the Azure tracing integration, used to filter traces.
27-
4) MODEL_DEPLOYMENT_NAME - Required. The Azure OpenAI deployment name to use with the built-in evaluators.
27+
4) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The Azure OpenAI deployment name to use with the built-in evaluators.
2828
5) TRACE_LOOKBACK_HOURS - Optional. Number of hours to look back when querying traces and in the evaluation run.
2929
Defaults to 1.
3030
"""
@@ -51,7 +51,7 @@
5151
"APPINSIGHTS_RESOURCE_ID"
5252
] # Sample : /subscriptions/<subscription_id>/resourceGroups/<rg_name>/providers/Microsoft.Insights/components/<resource_name>
5353
agent_id = os.environ["AGENT_ID"] # Sample : gcp-cloud-run-agent
54-
model_deployment_name = os.environ["MODEL_DEPLOYMENT_NAME"] # Sample : gpt-4o-mini
54+
model_deployment_name = os.environ["AZURE_AI_MODEL_DEPLOYMENT_NAME"] # Sample : gpt-4o-mini
5555
trace_query_hours = int(os.environ.get("TRACE_LOOKBACK_HOURS", "1"))
5656

5757

‎sdk/ai/azure-ai-projects/samples/evaluation/sample_evaluations_graders.py‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
It can be found in the Foundry overview page. It has the form https://<account_name>.openai.azure.com.
2525
4) MODEL_API_KEY - Required. The API key for the model endpoint. Can be found under "key" in the model details page
2626
(click "Models + endpoints" and select your model to get to the model details page).
27-
5) MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation.
27+
5) AZURE_AI_MODEL_DEPLOYMENT_NAME - Required. The name of the model deployment to use for evaluation.
2828
6) DATASET_NAME - Optional. The name of the Dataset to create and use in this sample.
2929
7) DATASET_VERSION - Optional. The version of the Dataset to create and use in this sample.
3030
8) DATA_FOLDER - Optional. The folder path where the data files for upload are located.
@@ -54,7 +54,7 @@
5454
connection_name = os.environ.get("CONNECTION_NAME", "")
5555
model_endpoint = os.environ.get("MODEL_ENDPOINT", "") # Sample: https://<account_name>.openai.azure.com.
5656
model_api_key = os.environ.get("MODEL_API_KEY", "")
57-
model_deployment_name = os.environ.get("MODEL_DEPLOYMENT_NAME", "") # Sample : gpt-4o-mini
57+
model_deployment_name = os.environ.get("AZURE_AI_MODEL_DEPLOYMENT_NAME", "") # Sample : gpt-4o-mini
5858
dataset_name = os.environ.get("DATASET_NAME", "")
5959
dataset_version = os.environ.get("DATASET_VERSION", "1")
6060

0 commit comments

Comments
 (0)