Implement custom user agent support in evaluate function

- Added user_agent parameter to evaluate() and _evaluate() functions
- Updated get_common_headers() to append custom user agent
- Created context variable system to pass user_agent to evaluators
- Updated all RAI service functions to support user_agent parameter
- Added docstring updates and usage examples
- Created sample demonstrating custom user agent usage
- Added unit tests for new functionality

Co-authored-by: singankit <[email protected]>

Author: Copilot

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_common/rai_service.py

@@ -0,0 +1,26 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+import contextvars
+from typing import Optional
+
+# Context variable to pass user_agent to evaluators
+_current_user_agent: contextvars.ContextVar[Optional[str]] = contextvars.ContextVar('current_user_agent', default=None)
+
+def get_current_user_agent() -> Optional[str]:
+    """Get the current user agent from context.
+    
+    :return: The current user agent if set, None otherwise.
+    :rtype: Optional[str]
+    """
+    return _current_user_agent.get()
+
+def set_current_user_agent(user_agent: Optional[str]) -> contextvars.Token:
+    """Set the current user agent in context.
+    
+    :param user_agent: The user agent to set.
+    :type user_agent: Optional[str]
+    :return: A token that can be used to reset the context.
+    :rtype: contextvars.Token
+    """
+    return _current_user_agent.set(user_agent)

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_context.py

@@ -31,6 +31,7 @@
 )
 from .._model_configurations import AzureAIProject, EvaluationResult, EvaluatorConfig
 from .._user_agent import USER_AGENT
+from .._context import set_current_user_agent
 from ._batch_run import (
     EvalRunContext,
     CodeClient,
@@ -54,7 +55,6 @@
     OAIEvalRunCreationInfo
 )
 LOGGER = logging.getLogger(__name__)
-
 # For metrics (aggregates) whose metric names intentionally differ from their
 # originating column name, usually because the aggregation of the original value
 # means something sufficiently different.
@@ -697,6 +697,7 @@ def evaluate(
     azure_ai_project: Optional[Union[str, AzureAIProject]] = None,
     output_path: Optional[Union[str, os.PathLike]] = None,
     fail_on_evaluator_errors: bool = False,
+    user_agent: Optional[str] = None,
     **kwargs,
 ) -> EvaluationResult:
     """Evaluates target or data with built-in or custom evaluators. If both target and data are provided,
@@ -728,6 +729,9 @@ def evaluate(
         Defaults to false, which means that evaluations will continue regardless of failures.
         If such failures occur, metrics may be missing, and evidence of failures can be found in the evaluation's logs.
     :paramtype fail_on_evaluator_errors: bool
+    :keyword user_agent: Custom user agent string to append to the default user agent for HTTP requests.
+        If provided, the final user agent will be: 'azure-ai-evaluation/<version> <user_agent>'.
+    :paramtype user_agent: Optional[str]
     :return: Evaluation results.
     :rtype: ~azure.ai.evaluation.EvaluationResult
 
@@ -749,6 +753,23 @@ def evaluate(
             :dedent: 8
             :caption: Run an evaluation on local data with one or more evaluators using Azure AI Project URL in following format 
                 https://{resource_name}.services.ai.azure.com/api/projects/{project_name}
+
+    .. admonition:: Example with custom user agent:
+
+        .. code-block:: python
+
+            from azure.ai.evaluation import evaluate
+            from azure.identity import DefaultAzureCredential
+
+            # Run evaluation with custom user agent
+            result = evaluate(
+                data="path/to/data.jsonl",
+                evaluators={
+                    "groundedness": GroundednessEvaluator(azure_ai_project=azure_ai_project, credential=credential)
+                },
+                azure_ai_project=azure_ai_project,
+                user_agent="MyApp/1.0.0"  # Custom user agent to append to default
+            )
     """
     try:
         return _evaluate(
@@ -760,6 +781,7 @@ def evaluate(
             azure_ai_project=azure_ai_project,
             output_path=output_path,
             fail_on_evaluator_errors=fail_on_evaluator_errors,
+            user_agent=user_agent,
             **kwargs,
         )
     except Exception as e:
@@ -828,11 +850,15 @@ def _evaluate(  # pylint: disable=too-many-locals,too-many-statements
     azure_ai_project: Optional[Union[str, AzureAIProject]] = None,
     output_path: Optional[Union[str, os.PathLike]] = None,
     fail_on_evaluator_errors: bool = False,
+    user_agent: Optional[str] = None,
     **kwargs,
 ) -> EvaluationResult:
     if fail_on_evaluator_errors:
         _print_fail_flag_warning()
 
+    # Set the user_agent in context for evaluators to access
+    set_current_user_agent(user_agent)
+    
     # Turn inputted mess of data into a dataframe, apply targets if needed
     # split graders and evaluators, and verify that column mappings are sensible.
     validated_data = _preprocess_data(

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluate/_evaluate.py

@@ -17,6 +17,7 @@
 from azure.ai.evaluation._common.utils import validate_conversation
 from azure.ai.evaluation._constants import _AggregationType
 from azure.core.credentials import TokenCredential
+from azure.ai.evaluation._context import get_current_user_agent
 
 from . import EvaluatorBase
 
@@ -122,6 +123,7 @@ async def _evaluate_conversation(self, conversation: Dict) -> Dict[str, T]:
             metric_name=self._eval_metric,
             project_scope=self._azure_ai_project,
             credential=self._credential,
+            user_agent=get_current_user_agent(),
         )
         return result
 
@@ -157,6 +159,7 @@ async def _evaluate_query_response(self, eval_input: Dict) -> Dict[str, T]:
             credential=self._credential,
             annotation_task=self._get_task(),
             evaluator_name=self.__class__.__name__,
+            user_agent=get_current_user_agent(),
         )
 
     def _get_task(self):

sdk/evaluation/azure-ai-evaluation/azure/ai/evaluation/_evaluators/_common/_base_rai_svc_eval.py

@@ -0,0 +1,68 @@
+# ---------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# ---------------------------------------------------------
+
+import json
+import os
+from azure.ai.evaluation import evaluate, GroundednessEvaluator
+from azure.identity import DefaultAzureCredential
+
+
+def main():
+    """Sample demonstrating how to use custom user agent with evaluate function."""
+    
+    # Setup Azure AI Project (replace with your actual project details)
+    azure_ai_project = {
+        "subscription_id": "your-subscription-id",
+        "resource_group_name": "your-resource-group",  
+        "project_name": "your-project-name",
+    }
+    credential = DefaultAzureCredential()
+    
+    # Create sample data
+    sample_data = [
+        {
+            "query": "What is the capital of France?",
+            "response": "The capital of France is Paris.",
+            "context": "France is a country in Western Europe. Its capital city is Paris, which is also the largest city in France."
+        },
+        {
+            "query": "What is machine learning?", 
+            "response": "Machine learning is a subset of artificial intelligence that uses algorithms to analyze data and make predictions.",
+            "context": "Artificial intelligence is a broad field that includes machine learning, deep learning, and other techniques for creating intelligent systems."
+        }
+    ]
+    
+    # Save sample data to a file
+    data_file = "sample_evaluation_data.jsonl"
+    with open(data_file, "w") as f:
+        for item in sample_data:
+            f.write(json.dumps(item) + "\n")
+    
+    try:
+        # [START evaluate_with_custom_user_agent]
+        # Run evaluation with custom user agent
+        result = evaluate(
+            data=data_file,
+            evaluators={
+                "groundedness": GroundednessEvaluator(
+                    azure_ai_project=azure_ai_project,
+                    credential=credential
+                )
+            },
+            azure_ai_project=azure_ai_project,
+            user_agent="MyApp/1.0.0 CustomEvaluation"  # Custom user agent to append to default
+        )
+        
+        print("Evaluation completed successfully!")
+        print(f"Results: {result}")
+        # [END evaluate_with_custom_user_agent]
+        
+    finally:
+        # Clean up sample data file
+        if os.path.exists(data_file):
+            os.remove(data_file)
+
+
+if __name__ == "__main__":
+    main()

sdk/evaluation/azure-ai-evaluation/samples/evaluation_custom_user_agent.py

@@ -963,4 +963,67 @@ def test_name_map_conversion(self):
         # Test failure case
         result = _convert_name_map_into_property_entries(test_map, segment_length=10, max_segments = 1)
         assert result[EvaluationRunProperties.NAME_MAP_LENGTH] == -1
-        assert len(result) == 1
+        assert len(result) == 1
+
+
+def test_evaluate_user_agent_parameter():
+    """Test that the user_agent parameter is accepted by the evaluate function."""
+    import tempfile
+    import json
+    from unittest.mock import patch, MagicMock
+    from azure.ai.evaluation._context import get_current_user_agent
+    
+    # Create test data
+    test_data = [
+        {"query": "What is the capital of France?", "response": "Paris"}
+    ]
+    
+    with tempfile.NamedTemporaryFile(mode='w', suffix='.jsonl', delete=False) as f:
+        for item in test_data:
+            f.write(json.dumps(item) + '\n')
+        temp_file = f.name
+    
+    try:
+        # Mock evaluator
+        def mock_evaluator(query, response):
+            # Check that user agent is available in context
+            user_agent = get_current_user_agent()
+            return {"score": 1.0, "user_agent_in_context": user_agent}
+        
+        # Test with custom user agent
+        with patch('azure.ai.evaluation._evaluate._evaluate._preprocess_data') as mock_preprocess, \
+             patch('azure.ai.evaluation._evaluate._evaluate._run_callable_evaluators') as mock_run_evaluators:
+            
+            mock_preprocess.return_value = {
+                "evaluators": {"test_evaluator": mock_evaluator},
+                "graders": {},
+                "input_data_df": pd.DataFrame(test_data),
+                "column_mapping": {},
+                "target_run": None,
+                "batch_run_client": MagicMock(),
+                "batch_run_data": temp_file
+            }
+            
+            mock_run_evaluators.return_value = (
+                pd.DataFrame([{"test_evaluator.score": 1.0}]),
+                {"test_evaluator.score": 1.0},
+                {}
+            )
+            
+            # Test that user_agent parameter is accepted and doesn't cause errors
+            result = evaluate(
+                data=temp_file,
+                evaluators={"test_evaluator": mock_evaluator},
+                user_agent="TestApp/1.0.0"
+            )
+            
+            # Verify the function was called
+            assert mock_preprocess.called
+            assert mock_run_evaluators.called
+            assert result is not None
+            
+    finally:
+        # Clean up
+        import os
+        if os.path.exists(temp_file):
+            os.unlink(temp_file)

sdk/evaluation/azure-ai-evaluation/tests/unittests/test_evaluate.py

@@ -0,0 +1,58 @@
+import pytest
+from azure.ai.evaluation._common.rai_service import get_common_headers
+
+
+def test_get_common_headers_default():
+    """Test get_common_headers with default parameters."""
+    token = "test_token"
+    headers = get_common_headers(token)
+    
+    assert "Authorization" in headers
+    assert headers["Authorization"] == "Bearer test_token"
+    assert "User-Agent" in headers
+    assert "azure-ai-evaluation" in headers["User-Agent"]
+
+
+def test_get_common_headers_with_evaluator_name():
+    """Test get_common_headers with evaluator name."""
+    token = "test_token"
+    evaluator_name = "TestEvaluator"
+    headers = get_common_headers(token, evaluator_name)
+    
+    assert "Authorization" in headers
+    assert headers["Authorization"] == "Bearer test_token"
+    assert "User-Agent" in headers
+    assert "azure-ai-evaluation" in headers["User-Agent"]
+    assert f"(type=evaluator; subtype={evaluator_name})" in headers["User-Agent"]
+
+
+def test_get_common_headers_with_custom_user_agent():
+    """Test get_common_headers with custom user agent."""
+    token = "test_token"
+    custom_user_agent = "MyApp/1.0.0"
+    headers = get_common_headers(token, user_agent=custom_user_agent)
+    
+    assert "Authorization" in headers
+    assert headers["Authorization"] == "Bearer test_token"
+    assert "User-Agent" in headers
+    assert "azure-ai-evaluation" in headers["User-Agent"]
+    assert custom_user_agent in headers["User-Agent"]
+    # Verify format: "azure-ai-evaluation/<version> MyApp/1.0.0"
+    assert headers["User-Agent"].endswith(f" {custom_user_agent}")
+
+
+def test_get_common_headers_with_evaluator_name_and_custom_user_agent():
+    """Test get_common_headers with both evaluator name and custom user agent."""
+    token = "test_token"
+    evaluator_name = "TestEvaluator"
+    custom_user_agent = "MyApp/1.0.0"
+    headers = get_common_headers(token, evaluator_name, custom_user_agent)
+    
+    assert "Authorization" in headers
+    assert headers["Authorization"] == "Bearer test_token"
+    assert "User-Agent" in headers
+    assert "azure-ai-evaluation" in headers["User-Agent"]
+    assert f"(type=evaluator; subtype={evaluator_name})" in headers["User-Agent"]
+    assert custom_user_agent in headers["User-Agent"]
+    # Verify format: "azure-ai-evaluation/<version> (type=evaluator; subtype=TestEvaluator) MyApp/1.0.0"
+    assert headers["User-Agent"].endswith(f" {custom_user_agent}")