[evaluation] feat: Allow users to update useragent (#41718)

* refactor: `USER_AGENT` constant -> UserAgentSingleton

    Moving to a singleton allows us to:

      * preserve similar ergonomics (import + use vs. forward through parameters)
      * allows us to dynamically change the useragent

* refactor: Refactor all uses of USER_AGENT constant to UserAgentSingleton().value

* fix: Set user agent

* feat: Add __str__ for UserAgentSingleton

* feat: Add a parameter to evaluate that allows users to set user_agent

* tests: Add tests for user-agent override

* tests,fix: Add missing recording fixtures

* tests,fix: Use "https://Sanitized.openai.azure.com" as sanitized endpoint

* tests: Update assets.json

* refactor: _user_agent -> user_agent

* fix,tests: Refactor model_config fixture

         Was previously relying on a local override of the mock_model_config
         fixture, to set azure_endpoint to a value that wouldn't
         cause a recording conflict.

         This worked locally when exclusively running the tests that
         needed the overriden value.

         In CI, the override would fail and the relevant tests failed.

         I *suspect* this is essentially a race condition caused by
         fixture scope. "Session" scoped fixtures are only instantiated
         once per session. So whether or not the override applies depends
         on whether the normal fixture or the overriden one is requested
         first. This hasn't been validated, but seems plausible.

         This entire mess can probably be avoided by just using
         a fixture with a different name.

* chore: Update assets.json

Author: kdestin

sdk/evaluation/azure-ai-evaluation/assets.json

@@ -2,5 +2,5 @@
   "AssetsRepo": "Azure/azure-sdk-assets",
   "AssetsRepoPrefixPath": "python",
   "TagPrefix": "python/evaluation/azure-ai-evaluation",
-  "Tag": "python/evaluation/azure-ai-evaluation_64eea0ea34"
+  "Tag": "python/evaluation/azure-ai-evaluation_e9c7adf5b1"
 }

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_aoai/aoai_grader.py

@@ -5,6 +5,7 @@
 
 from azure.ai.evaluation._constants import DEFAULT_AOAI_API_VERSION
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
+from azure.ai.evaluation._user_agent import UserAgentSingleton
 from typing import Any, Dict, Union
 from azure.ai.evaluation._common._experimental import experimental
 
@@ -71,6 +72,9 @@ def get_client(self) -> Any:
         :return: The OpenAI client.
         :rtype: [~openai.OpenAI, ~openai.AzureOpenAI]
         """
+        default_headers = {
+            "User-Agent": UserAgentSingleton().value
+        }
         if "azure_endpoint" in self._model_config:
            from openai import AzureOpenAI
            # TODO set default values?
@@ -79,11 +83,13 @@ def get_client(self) -> Any:
                 api_key=self._model_config.get("api_key", None), # Default-style access to appease linters.
                 api_version=DEFAULT_AOAI_API_VERSION, # Force a known working version
                 azure_deployment=self._model_config.get("azure_deployment", ""),
+                default_headers=default_headers
             )
         from openai import OpenAI
         # TODO add default values for base_url and organization?
         return OpenAI(
             api_key=self._model_config["api_key"],
             base_url=self._model_config.get("base_url", ""),
             organization=self._model_config.get("organization", ""),
+            default_headers=default_headers
         )

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py

@@ -21,10 +21,11 @@
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._http_utils import AsyncHttpPipeline, get_async_http_client
 from azure.ai.evaluation._model_configurations import AzureAIProject
+from azure.ai.evaluation._user_agent import UserAgentSingleton
 from azure.ai.evaluation._common.utils import is_onedp_project
 from azure.core.credentials import TokenCredential
 from azure.core.exceptions import HttpResponseError
-from azure.core.pipeline.policies import AsyncRetryPolicy
+from azure.core.pipeline.policies import AsyncRetryPolicy, UserAgentPolicy
 
 from .constants import (
     CommonConstants,
@@ -35,11 +36,6 @@
 )
 from .utils import get_harm_severity_level, retrieve_content_type
 
-try:
-    version = importlib.metadata.version("azure-ai-evaluation")
-except importlib.metadata.PackageNotFoundError:
-    version = "unknown"
-USER_AGENT = "{}/{}".format("azure-ai-evaluation", version)
 
 USER_TEXT_TEMPLATE_DICT: Dict[str, Template] = {
     "DEFAULT": Template("<Human>{$query}</><System>{$response}</>"),
@@ -101,7 +97,7 @@ def get_common_headers(token: str, evaluator_name: Optional[str] = None) -> Dict
     :return: The common headers.
     :rtype: Dict
     """
-    user_agent = f"{USER_AGENT} (type=evaluator; subtype={evaluator_name})" if evaluator_name else USER_AGENT
+    user_agent = f"{UserAgentSingleton().value} (type=evaluator; subtype={evaluator_name})" if evaluator_name else UserAgentSingleton().value
     return {
         "Authorization": f"Bearer {token}",
         "User-Agent": user_agent,
@@ -645,7 +641,7 @@ async def evaluate_with_rai_service(
     """
 
     if is_onedp_project(project_scope):
-        client = AIProjectClient(endpoint=project_scope, credential=credential)
+        client = AIProjectClient(endpoint=project_scope, credential=credential, user_agent_policy=UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
         token = await fetch_or_reuse_token(credential=credential, workspace=COG_SRV_WORKSPACE)
         await ensure_service_availability_onedp(client, token, annotation_task)
         operation_id = await submit_request_onedp(client, data, metric_name, token, annotation_task, evaluator_name)
@@ -788,7 +784,7 @@ async def evaluate_with_rai_service_multimodal(
     """
 
     if is_onedp_project(project_scope):
-        client = AIProjectClient(endpoint=project_scope, credential=credential)
+        client = AIProjectClient(endpoint=project_scope, credential=credential, user_agent_policy=UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
         token = await fetch_or_reuse_token(credential=credential, workspace=COG_SRV_WORKSPACE)
         await ensure_service_availability_onedp(client, token, Tasks.CONTENT_HARM)
         operation_id = await submit_multimodal_request_onedp(client, messages, metric_name, token)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_batch_run/eval_run_context.py

@@ -21,7 +21,7 @@
     PF_DISABLE_TRACING,
 )
 
-from ..._user_agent import USER_AGENT
+from ..._user_agent import UserAgentSingleton
 from .._utils import set_event_loop_policy
 from .batch_clients import BatchClient
 from ._run_submitter_client import RunSubmitterClient
@@ -50,7 +50,7 @@ def __enter__(self) -> None:
         self._original_cwd = os.getcwd()
 
         if isinstance(self.client, CodeClient):
-            ClientUserAgentUtil.append_user_agent(USER_AGENT)
+            ClientUserAgentUtil.append_user_agent(UserAgentSingleton().value)
             inject_openai_api()
 
         if isinstance(self.client, ProxyClient):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py

@@ -2,6 +2,7 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # ---------------------------------------------------------
 import inspect
+import contextlib
 import json
 import logging
 import os
@@ -30,7 +31,7 @@
     DEFAULT_OAI_EVAL_RUN_NAME
 )
 from .._model_configurations import AzureAIProject, EvaluationResult, EvaluatorConfig
-from .._user_agent import USER_AGENT
+from .._user_agent import UserAgentSingleton
 from ._batch_run import (
     EvalRunContext,
     CodeClient,
@@ -729,6 +730,8 @@ def evaluate(
         Defaults to false, which means that evaluations will continue regardless of failures.
         If such failures occur, metrics may be missing, and evidence of failures can be found in the evaluation's logs.
     :paramtype fail_on_evaluator_errors: bool
+    :keyword user_agent: A string to append to the default user-agent sent with evaluation http requests
+    :paramtype user_agent: Optional[str]
     :return: Evaluation results.
     :rtype: ~azure.ai.evaluation.EvaluationResult
 
@@ -752,17 +755,19 @@ def evaluate(
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
     """
     try:
-        return _evaluate(
-            evaluation_name=evaluation_name,
-            target=target,
-            data=data,
-            evaluators_and_graders=evaluators,
-            evaluator_config=evaluator_config,
-            azure_ai_project=azure_ai_project,
-            output_path=output_path,
-            fail_on_evaluator_errors=fail_on_evaluator_errors,
-            **kwargs,
-        )
+        user_agent: Optional[str] = kwargs.get("user_agent")
+        with UserAgentSingleton().add_useragent_product(user_agent) if user_agent else contextlib.nullcontext():
+            return _evaluate(
+                evaluation_name=evaluation_name,
+                target=target,
+                data=data,
+                evaluators_and_graders=evaluators,
+                evaluator_config=evaluator_config,
+                azure_ai_project=azure_ai_project,
+                output_path=output_path,
+                fail_on_evaluator_errors=fail_on_evaluator_errors,
+                **kwargs,
+            )
     except Exception as e:
         # Handle multiprocess bootstrap error
         bootstrap_error = (
@@ -997,7 +1002,7 @@ def _preprocess_data(
         batch_run_client = RunSubmitterClient()
         batch_run_data = input_data_df
     elif kwargs.pop("_use_pf_client", True):
-        batch_run_client = ProxyClient(user_agent=USER_AGENT)
+        batch_run_client = ProxyClient(user_agent=UserAgentSingleton().value)
         # Ensure the absolute path is passed to pf.run, as relative path doesn't work with
         # multiple evaluators. If the path is already absolute, abspath will return the original path.
         batch_run_data = os.path.abspath(data)
@@ -1179,4 +1184,4 @@ def _turn_error_logs_into_exception(log_path: str) -> None:
         target=ErrorTarget.EVALUATE,
         category=ErrorCategory.FAILED_EXECUTION,
         blame=ErrorBlame.UNKNOWN,
-    )
+    )

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_telemetry/__init__.py

@@ -17,7 +17,6 @@
 
 from azure.ai.evaluation._model_configurations import AzureAIProject, EvaluationResult
 
-from ..._user_agent import USER_AGENT
 from .._utils import _trace_destination_from_project_scope
 
 LOGGER = logging.getLogger(__name__)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_utils.py

@@ -15,6 +15,7 @@
 import pandas as pd
 from tqdm import tqdm
 
+from azure.core.pipeline.policies import UserAgentPolicy
 from azure.ai.evaluation._legacy._adapters.entities import Run
 
 from azure.ai.evaluation._constants import (
@@ -26,6 +27,7 @@
 from azure.ai.evaluation._exceptions import ErrorBlame, ErrorCategory, ErrorTarget, EvaluationException
 from azure.ai.evaluation._model_configurations import AzureAIProject
 from azure.ai.evaluation._version import VERSION
+from azure.ai.evaluation._user_agent import UserAgentSingleton
 from azure.ai.evaluation._azure._clients import LiteMLClient
 
 LOGGER = logging.getLogger(__name__)
@@ -148,7 +150,8 @@ def _log_metrics_and_instance_results_onedp(
     )
     client = EvaluationServiceOneDPClient(
         endpoint=project_url,
-        credential=credentials
+        credential=credentials,
+        user_agent_policy=UserAgentPolicy(base_user_agent=UserAgentSingleton().value)
     )
 
     # Massaging before artifacts are put on disk

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_prompty_eval.py

@@ -20,9 +20,12 @@
 from . import EvaluatorBase
 
 try:
-    from ..._user_agent import USER_AGENT
+    from ..._user_agent import UserAgentSingleton
 except ImportError:
-    USER_AGENT = "None"
+    class UserAgentSingleton:
+        @property
+        def value(self) -> str:
+            return "None"
 
 T = TypeVar("T")
 
@@ -60,7 +63,7 @@ def __init__(self, *, result_key: str, prompty_file: str, model_config: dict, ev
         super().__init__(eval_last_turn=eval_last_turn, threshold=threshold, _higher_is_better=_higher_is_better)
 
         subclass_name = self.__class__.__name__
-        user_agent = f"{USER_AGENT} (type=evaluator subtype={subclass_name})"
+        user_agent = f"{UserAgentSingleton().value} (type=evaluator subtype={subclass_name})"
         prompty_model_config = construct_prompty_model_config(
             validate_model_config(model_config),
             self._DEFAULT_OPEN_API_VERSION,

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_groundedness/_groundedness.py

@@ -12,9 +12,12 @@
 from ..._common.utils import construct_prompty_model_config, validate_model_config
 
 try:
-    from ..._user_agent import USER_AGENT
+    from ..._user_agent import UserAgentSingleton
 except ImportError:
-    USER_AGENT = "None"
+    class UserAgentSingleton:
+        @property
+        def value(self) -> str:
+            return "None"
 
 
 class GroundednessEvaluator(PromptyEvaluatorBase[Union[str, float]]):
@@ -165,7 +168,7 @@ def __call__(  # pylint: disable=docstring-missing-param
             prompty_model_config = construct_prompty_model_config(
                 validate_model_config(self._model_config),
                 self._DEFAULT_OPEN_API_VERSION,
-                USER_AGENT,
+                UserAgentSingleton().value,
             )
             self._flow = AsyncPrompty.load(source=self._prompty_file, model=prompty_model_config)
 

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_http_utils.py

@@ -7,7 +7,7 @@
 
 from typing_extensions import Self, Unpack
 
-from azure.ai.evaluation._user_agent import USER_AGENT
+from azure.ai.evaluation._user_agent import UserAgentSingleton
 from azure.core.configuration import Configuration
 from azure.core.pipeline import AsyncPipeline, Pipeline
 from azure.core.pipeline.policies import (
@@ -454,7 +454,7 @@ def get_http_client(**kwargs: Any) -> HttpPipeline:
     :returns: An HttpPipeline with a set of applied policies:
     :rtype: HttpPipeline
     """
-    kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=USER_AGENT))
+    kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
     return HttpPipeline(**kwargs)
 
 
@@ -464,5 +464,5 @@ def get_async_http_client(**kwargs: Any) -> AsyncHttpPipeline:
     :returns: An AsyncHttpPipeline with a set of applied policies:
     :rtype: AsyncHttpPipeline
     """
-    kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=USER_AGENT))
+    kwargs.setdefault("user_agent_policy", UserAgentPolicy(base_user_agent=UserAgentSingleton().value))
     return AsyncHttpPipeline(**kwargs)