Skip to content

Commit e797804

Browse files
singankitsingankit
authored
Add Evaluation Service 1DP Client (Azure#40683)
1 parent fadb096 commit e797804

File tree

3 files changed

+180
-1
lines changed

3 files changed

+180
-1
lines changed

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/evaluation_onedp_client.py‎

Lines changed: 115 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,115 @@
1+
# ---------------------------------------------------------
2+
# Copyright (c) Microsoft Corporation. All rights reserved.
3+
# ---------------------------------------------------------
4+
5+
import logging
6+
from typing import Union, Any
7+
from azure.core.credentials import AzureKeyCredential, TokenCredential
8+
from azure.ai.evaluation._common.onedp import AIProjectClient as RestEvaluationServiceClient
9+
from azure.ai.evaluation._common.onedp.models import (PendingUploadRequest, PendingUploadType, EvaluationResult,
10+
ResultType, AssetCredentialRequest, EvaluationUpload, InputDataset)
11+
from azure.storage.blob import ContainerClient
12+
from .utils import upload
13+
14+
LOGGER = logging.getLogger(__name__)
15+
16+
class EvaluationServiceOneDPClient:
17+
18+
def __init__(self, endpoint: str, credential: Union[AzureKeyCredential, "TokenCredential"], **kwargs: Any) -> None:
19+
self.rest_client = RestEvaluationServiceClient(
20+
endpoint=endpoint,
21+
credential=credential,
22+
**kwargs,
23+
)
24+
25+
def create_evaluation_result(self, *, name: str, path: str, version=1, **kwargs) -> None:
26+
"""Create and upload evaluation results to Azure evaluation service.
27+
28+
This method uploads evaluation results from a local path to Azure Blob Storage
29+
and registers them with the evaluation service. The process involves:
30+
1. Starting a pending upload with the evaluation service
31+
2. Getting a SAS token for the blob container
32+
3. Uploading the local evaluation results to the blob container
33+
4. Creating a version record for the evaluation results
34+
35+
:param name: The name to identify the evaluation results
36+
:type name: str
37+
:param path: The local path to the evaluation results file or directory
38+
:type path: str
39+
:param version: The version number for the evaluation results, defaults to 1
40+
:type version: int, optional
41+
:param kwargs: Additional keyword arguments to pass to the underlying API calls
42+
:return: The response from creating the evaluation result version
43+
:rtype: EvaluationResult
44+
:raises: Various exceptions from the underlying API calls or upload process
45+
"""
46+
47+
LOGGER.debug(f"Creating evaluation result for {name} with version {version} from path {path}")
48+
start_pending_upload_response = self.rest_client.evaluation_results.start_pending_upload(
49+
name=name,
50+
version=version,
51+
body=PendingUploadRequest(pending_upload_type=PendingUploadType.TEMPORARY_BLOB_REFERENCE),
52+
**kwargs
53+
)
54+
55+
LOGGER.debug(f"Uploading {path} to {start_pending_upload_response.blob_reference_for_consumption.blob_uri}")
56+
with ContainerClient.from_container_url(
57+
start_pending_upload_response.blob_reference_for_consumption.credential.sas_uri) as container_client:
58+
upload(path=path, container_client=container_client, logger=LOGGER)
59+
60+
LOGGER.debug(f"Creating evaluation result version for {name} with version {version}")
61+
create_version_response = self.rest_client.evaluation_results.create_version(
62+
body=EvaluationResult(
63+
blob_uri=start_pending_upload_response.blob_reference_for_consumption.blob_uri,
64+
result_type=ResultType.EVALUATION,
65+
name=name,
66+
version=version
67+
),
68+
name=name,
69+
version=version,
70+
**kwargs
71+
)
72+
73+
return create_version_response
74+
75+
def start_evaluation_run(self, *, evaluation: EvaluationUpload, **kwargs) -> EvaluationUpload:
76+
"""Start a new evaluation run in the Azure evaluation service.
77+
78+
This method creates a new evaluation run with the provided configuration details.
79+
80+
:param evaluation: The evaluation configuration to upload
81+
:type evaluation: EvaluationUpload
82+
:param kwargs: Additional keyword arguments to pass to the underlying API calls
83+
:return: The created evaluation run object
84+
:rtype: EvaluationUpload
85+
:raises: Various exceptions from the underlying API calls
86+
"""
87+
upload_run_response = self.rest_client.evaluations.upload_run(
88+
evaluation=evaluation,
89+
**kwargs
90+
)
91+
92+
return upload_run_response
93+
94+
def update_evaluation_run(self, *, name: str, evaluation: EvaluationUpload, **kwargs) -> EvaluationUpload:
95+
"""Update an existing evaluation run in the Azure evaluation service.
96+
97+
This method updates an evaluation run with new information such as status changes,
98+
result references, or other metadata.
99+
100+
:param name: The identifier of the evaluation run to update
101+
:type name: str
102+
:param evaluation: The updated evaluation configuration
103+
:type evaluation: EvaluationUpload
104+
:param kwargs: Additional keyword arguments to pass to the underlying API calls
105+
:return: The updated evaluation run object
106+
:rtype: EvaluationUpload
107+
:raises: Various exceptions from the underlying API calls
108+
"""
109+
update_run_response = self.rest_client.evaluations.upload_update_run(
110+
name=name,
111+
evaluation=evaluation,
112+
**kwargs
113+
)
114+
115+
return update_run_response

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/utils.py‎

Lines changed: 64 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,13 +1,15 @@
11
# ---------------------------------------------------------
22
# Copyright (c) Microsoft Corporation. All rights reserved.
33
# ---------------------------------------------------------
4-
4+
import os
5+
import posixpath
56
import re
67
import math
78
import threading
89
from typing import Any, List, Literal, Mapping, Type, TypeVar, Tuple, Union, cast, get_args, get_origin
910

1011
import nltk
12+
from azure.storage.blob import ContainerClient
1113
from typing_extensions import NotRequired, Required, TypeGuard
1214
from azure.ai.evaluation._legacy._adapters._errors import MissingRequiredPackage
1315
from azure.ai.evaluation._constants import AZURE_OPENAI_TYPE, OPENAI_TYPE
@@ -463,3 +465,64 @@ def raise_exception(msg, target):
463465
"User and assistant role expected as the only role in each message.",
464466
ErrorTarget.CONTENT_SAFETY_CHAT_EVALUATOR,
465467
)
468+
469+
def upload(path: str, container_client: ContainerClient, logger=None):
470+
"""Upload files or directories to Azure Blob Storage using a container client.
471+
472+
This function uploads a file or all files in a directory (recursively) to Azure Blob Storage.
473+
When uploading a directory, the relative path structure is preserved in the blob container.
474+
475+
:param path: The local path to a file or directory to upload
476+
:type path: str
477+
:param container_client: The Azure Blob Container client to use for uploading
478+
:type container_client: azure.storage.blob.ContainerClient
479+
:param logger: Optional logger for debug output, defaults to None
480+
:type logger: logging.Logger, optional
481+
:raises EvaluationException: If the path doesn't exist or errors occur during upload
482+
"""
483+
484+
if not os.path.isdir(path) and not os.path.isfile(path):
485+
raise EvaluationException(
486+
message=f"Path '{path}' is not a directory or a file",
487+
internal_message=f"Path '{path}' is not a directory or a file",
488+
target=ErrorTarget.RAI_CLIENT,
489+
category=ErrorCategory.INVALID_VALUE,
490+
blame=ErrorBlame.SYSTEM_ERROR,
491+
)
492+
493+
remote_paths = []
494+
local_paths = []
495+
496+
if os.path.isdir(path):
497+
for (root, _, filenames) in os.walk(path):
498+
upload_path = ""
499+
if root != path:
500+
rel_path = os.path.relpath(root, path)
501+
upload_path = posixpath.join(rel_path)
502+
for f in filenames:
503+
remote_file_path = posixpath.join(upload_path, f)
504+
remote_paths.append(remote_file_path)
505+
local_file_path = os.path.join(root, f)
506+
local_paths.append(local_file_path)
507+
508+
if os.path.isfile(path):
509+
remote_paths = [os.path.basename(path)]
510+
local_paths = [path]
511+
512+
try:
513+
# Open the file in binary read mode
514+
for local, remote in zip(local_paths, remote_paths):
515+
with open(local, "rb") as data:
516+
# Upload the file to Azure Blob Storage
517+
container_client.upload_blob(data=data, name=remote)
518+
if logger:
519+
logger.debug(f"File '{local}' uploaded successfully")
520+
521+
except Exception as e:
522+
raise EvaluationException(
523+
message=f"Error uploading file: {e}",
524+
internal_message=f"Error uploading file: {e}",
525+
target=ErrorTarget.RAI_CLIENT,
526+
category=ErrorCategory.UPLOAD_ERROR,
527+
blame=ErrorBlame.SYSTEM_ERROR,
528+
)

‎sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_exceptions.py‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -38,6 +38,7 @@ class ErrorCategory(Enum):
3838
FAILED_REMOTE_TRACKING = "FAILED REMOTE TRACKING"
3939
PROJECT_ACCESS_ERROR = "PROJECT ACCESS ERROR"
4040
UNKNOWN = "UNKNOWN"
41+
UPLOAD_ERROR = "UPLOAD ERROR"
4142

4243

4344
class ErrorBlame(Enum):

0 commit comments

Comments
 (0)