rename

Author: nina-kollman

packages/traceloop-sdk/tests/evaluator/test_evaluator.py

@@ -1,5 +1,5 @@
 import pytest
-from traceloop.sdk.evaluator.evaluator import validate_task_output
+from traceloop.sdk.evaluator.evaluator import validate_and_normalize_task_output
 from traceloop.sdk.evaluator.config import EvaluatorDetails
 
 
@@ -12,7 +12,7 @@ def test_validate_task_output_with_no_evaluators(self):
         evaluators = []
 
         # Should not raise any exception
-        validate_task_output(task_output, evaluators)
+        validate_and_normalize_task_output(task_output, evaluators)
 
     def test_validate_task_output_with_evaluators_no_required_fields(self):
         """Test that validation passes when evaluators have no required fields"""
@@ -23,7 +23,7 @@ def test_validate_task_output_with_evaluators_no_required_fields(self):
         ]
 
         # Should not raise any exception
-        validate_task_output(task_output, evaluators)
+        validate_and_normalize_task_output(task_output, evaluators)
 
     def test_validate_task_output_with_valid_output(self):
         """Test that validation passes when all required fields are present"""
@@ -37,7 +37,7 @@ def test_validate_task_output_with_valid_output(self):
         ]
 
         # Should not raise any exception
-        validate_task_output(task_output, evaluators)
+        validate_and_normalize_task_output(task_output, evaluators)
 
     def test_validate_task_output_missing_single_field(self):
         """Test that validation fails when a single required field is missing"""
@@ -47,7 +47,7 @@ def test_validate_task_output_missing_single_field(self):
         ]
 
         with pytest.raises(ValueError) as exc_info:
-            validate_task_output(task_output, evaluators)
+            validate_and_normalize_task_output(task_output, evaluators)
 
         error_message = str(exc_info.value)
         assert "Task output missing required fields for evaluators:" in error_message
@@ -70,7 +70,7 @@ def test_validate_task_output_missing_multiple_fields_single_evaluator(self):
         ]
 
         with pytest.raises(ValueError) as exc_info:
-            validate_task_output(task_output, evaluators)
+            validate_and_normalize_task_output(task_output, evaluators)
 
         error_message = str(exc_info.value)
         assert "relevance-checker requires:" in error_message
@@ -91,7 +91,7 @@ def test_validate_task_output_missing_fields_multiple_evaluators(self):
         ]
 
         with pytest.raises(ValueError) as exc_info:
-            validate_task_output(task_output, evaluators)
+            validate_and_normalize_task_output(task_output, evaluators)
 
         error_message = str(exc_info.value)
         assert "pii-detector requires:" in error_message
@@ -113,7 +113,7 @@ def test_validate_task_output_partial_match(self):
         ]
 
         with pytest.raises(ValueError) as exc_info:
-            validate_task_output(task_output, evaluators)
+            validate_and_normalize_task_output(task_output, evaluators)
 
         error_message = str(exc_info.value)
         # Should only mention the failing evaluator
@@ -128,7 +128,7 @@ def test_validate_task_output_empty_task_output(self):
         ]
 
         with pytest.raises(ValueError) as exc_info:
-            validate_task_output(task_output, evaluators)
+            validate_and_normalize_task_output(task_output, evaluators)
 
         error_message = str(exc_info.value)
         assert "Task output contains: []" in error_message
@@ -147,7 +147,7 @@ def test_validate_task_output_with_extra_fields(self):
         ]
 
         # Should not raise any exception - extra fields are allowed
-        validate_task_output(task_output, evaluators)
+        validate_and_normalize_task_output(task_output, evaluators)
 
     def test_validate_task_output_case_sensitive_field_names(self):
         """Test that field name matching is case-sensitive"""
@@ -157,7 +157,7 @@ def test_validate_task_output_case_sensitive_field_names(self):
         ]
 
         with pytest.raises(ValueError) as exc_info:
-            validate_task_output(task_output, evaluators)
+            validate_and_normalize_task_output(task_output, evaluators)
 
         error_message = str(exc_info.value)
         assert "pii-detector requires:" in error_message
@@ -177,7 +177,7 @@ def test_validate_task_output_with_evaluator_config(self):
         ]
 
         # Should not raise any exception - config shouldn't affect validation
-        validate_task_output(task_output, evaluators)
+        validate_and_normalize_task_output(task_output, evaluators)
 
     def test_validate_task_output_mixed_evaluators(self):
         """Test validation with a mix of evaluators with and without required fields"""
@@ -193,7 +193,7 @@ def test_validate_task_output_mixed_evaluators(self):
         ]
 
         with pytest.raises(ValueError) as exc_info:
-            validate_task_output(task_output, evaluators)
+            validate_and_normalize_task_output(task_output, evaluators)
 
         error_message = str(exc_info.value)
         # Should only mention failing evaluator
@@ -215,7 +215,7 @@ def test_validate_task_output_duplicate_required_fields(self):
         ]
 
         with pytest.raises(ValueError) as exc_info:
-            validate_task_output(task_output, evaluators)
+            validate_and_normalize_task_output(task_output, evaluators)
 
         error_message = str(exc_info.value)
         assert "pii-detector requires:" in error_message

packages/traceloop-sdk/tests/evaluator/test_field_mapping.py

@@ -328,7 +328,7 @@ class TestIntegrationWithValidateTaskOutput:
 
     def test_validate_with_synonym_mapping(self):
         """Test that validate_task_output uses synonym mapping"""
-        from traceloop.sdk.evaluator.evaluator import validate_task_output
+        from traceloop.sdk.evaluator.evaluator import validate_and_normalize_task_output
         from traceloop.sdk.evaluator.config import EvaluatorDetails
 
         # User returns "answer" but evaluator needs "completion"
@@ -341,15 +341,15 @@ def test_validate_with_synonym_mapping(self):
         ]
 
         # Should not raise - synonyms should be mapped
-        normalized = validate_task_output(task_output, evaluators)
+        normalized = validate_and_normalize_task_output(task_output, evaluators)
         assert "completion" in normalized
         assert "question" in normalized
         assert normalized["completion"] == "Paris"
         assert normalized["question"] == "What is the capital?"
 
     def test_validate_fails_with_helpful_message(self):
         """Test that validation failure includes synonym suggestions"""
-        from traceloop.sdk.evaluator.evaluator import validate_task_output
+        from traceloop.sdk.evaluator.evaluator import validate_and_normalize_task_output
         from traceloop.sdk.evaluator.config import EvaluatorDetails
 
         task_output = {"wrong_field": "value"}
@@ -361,7 +361,7 @@ def test_validate_fails_with_helpful_message(self):
         ]
 
         with pytest.raises(ValueError) as exc_info:
-            validate_task_output(task_output, evaluators)
+            validate_and_normalize_task_output(task_output, evaluators)
 
         error_message = str(exc_info.value)
         assert "test-evaluator requires:" in error_message
@@ -370,7 +370,7 @@ def test_validate_fails_with_helpful_message(self):
 
     def test_validate_with_context_to_reference_mapping(self):
         """Test specific case of context mapping to reference"""
-        from traceloop.sdk.evaluator.evaluator import validate_task_output
+        from traceloop.sdk.evaluator.evaluator import validate_and_normalize_task_output
         from traceloop.sdk.evaluator.config import EvaluatorDetails
 
         task_output = {
@@ -385,14 +385,14 @@ def test_validate_with_context_to_reference_mapping(self):
             )
         ]
 
-        normalized = validate_task_output(task_output, evaluators)
+        normalized = validate_and_normalize_task_output(task_output, evaluators)
         assert normalized["completion"] == "Yes"
         assert normalized["question"] == "Is it true?"
         assert normalized["reference"] == "The sky is blue"
 
     def test_validate_with_trajectory_fields(self):
         """Test mapping for trajectory fields used in agent evaluators"""
-        from traceloop.sdk.evaluator.evaluator import validate_task_output
+        from traceloop.sdk.evaluator.evaluator import validate_and_normalize_task_output
         from traceloop.sdk.evaluator.config import EvaluatorDetails
 
         task_output = {
@@ -406,6 +406,6 @@ def test_validate_with_trajectory_fields(self):
             )
         ]
 
-        normalized = validate_task_output(task_output, evaluators)
+        normalized = validate_and_normalize_task_output(task_output, evaluators)
         assert normalized["trajectory_prompts"] == "prompt1, prompt2"
         assert normalized["trajectory_completions"] == "comp1, comp2"

packages/traceloop-sdk/traceloop/sdk/evaluator/evaluator.py

@@ -1,5 +1,6 @@
 import httpx
 from typing import Dict, Optional, Any, List
+from .field_mapping import normalize_task_output, get_field_suggestions, format_field_help
 
 from .model import (
     InputExtractor,
@@ -145,7 +146,7 @@ async def trigger_experiment_evaluator(
         return execute_response.execution_id
 
 
-def validate_task_output(
+def validate_and_normalize_task_output(
     task_output: Dict[str, Any],
     evaluators: List[EvaluatorDetails],
 ) -> Dict[str, Any]:
@@ -163,8 +164,6 @@ def validate_task_output(
     Raises:
         ValueError: If task output is missing required fields for any evaluator (even after synonym mapping)
     """
-    from .field_mapping import normalize_task_output, get_field_suggestions, format_field_help
-
     if not evaluators:
         return task_output
 

packages/traceloop-sdk/traceloop/sdk/experiment/experiment.py

@@ -5,7 +5,7 @@
 from typing import Any, List, Callable, Optional, Tuple, Dict, Awaitable, Union
 from traceloop.sdk.client.http import HTTPClient
 from traceloop.sdk.datasets.datasets import Datasets
-from traceloop.sdk.evaluator.evaluator import Evaluator, validate_task_output
+from traceloop.sdk.evaluator.evaluator import Evaluator, validate_and_normalize_task_output
 from traceloop.sdk.experiment.model import (
     InitExperimentRequest,
     ExperimentInitResponse,
@@ -168,7 +168,7 @@ async def run_single_row(row: Optional[Dict[str, Any]]) -> TaskResponse:
 
                 # Validate task output with EvaluatorDetails and normalize field names using synonyms
                 if evaluators_to_validate:
-                    task_result = validate_task_output(task_result, evaluators_to_validate)
+                    task_result = validate_and_normalize_task_output(task_result, evaluators_to_validate)
 
                 task_id = self._create_task(
                     experiment_slug=experiment_slug,
@@ -484,7 +484,7 @@ async def run_single_row(row: Optional[Dict[str, Any]]) -> TaskResult:
                 # Validate task output schema and normalize field names using synonyms
                 if evaluators_to_validate:
                     try:
-                        task_output = validate_task_output(task_output, evaluators_to_validate)
+                        task_output = validate_and_normalize_task_output(task_output, evaluators_to_validate)
                     except ValueError as validation_error:
                         print(f"\033[91m❌ Task validation failed: {str(validation_error)}\033[0m")
                         raise ValueError(str(validation_error))