Remove cases and averages from eval span (#2715)

Co-authored-by: David Montague <[email protected]>

Author: DouweM

pydantic_evals/pydantic_evals/dataset.py

@@ -282,7 +282,10 @@ async def evaluate(
 
         limiter = anyio.Semaphore(max_concurrency) if max_concurrency is not None else AsyncExitStack()
 
-        with _logfire.span('evaluate {name}', name=name) as eval_span, progress_bar or nullcontext():
+        with (
+            _logfire.span('evaluate {name}', name=name, n_cases=len(self.cases)) as eval_span,
+            progress_bar or nullcontext(),
+        ):
             task_id = progress_bar.add_task(f'Evaluating {name}', total=total_cases) if progress_bar else None
 
             async def _handle_case(case: Case[InputsT, OutputT, MetadataT], report_case_name: str):
@@ -320,15 +323,8 @@ async def _handle_case(case: Case[InputsT, OutputT, MetadataT], report_case_name
                 span_id=span_id,
                 trace_id=trace_id,
             )
-            # TODO(DavidM): Address the following TODOs before V1...
-            # TODO(DavidM): This attribute will be too big in general; remove it once we can use child spans in details panel:
-            eval_span.set_attribute('cases', _REPORT_CASES_ADAPTER.dump_python(report.cases))
-            # TODO(DavidM): This attribute will be too big in general; remove it once we can use child spans in details panel:
-            eval_span.set_attribute('failures', _REPORT_CASE_FAILURES_ADAPTER.dump_python(report.failures))
-            # TODO(DavidM): Remove this 'averages' attribute once we compute it in the details panel
-            averages = report.averages()
-            if averages:
-                eval_span.set_attribute('averages', _REPORT_CASE_AGGREGATE_ADAPTER.dump_python(averages))
+            if (averages := report.averages()) is not None and averages.assertions is not None:
+                eval_span.set_attribute('assertion_pass_rate', averages.assertions)
         return report
 
     def evaluate_sync(

tests/evals/test_dataset.py

@@ -1482,91 +1482,14 @@ async def mock_async_task(inputs: TaskInput) -> TaskOutput:
                 'evaluate {name}',
                 {
                     'name': 'mock_async_task',
+                    'n_cases': 2,
+                    'assertion_pass_rate': 1.0,
                     'logfire.msg_template': 'evaluate {name}',
                     'logfire.msg': 'evaluate mock_async_task',
                     'logfire.span_type': 'span',
-                    'cases': [
-                        {
-                            'name': 'case1',
-                            'inputs': {'query': 'What is 2+2?'},
-                            'metadata': {'difficulty': 'easy', 'category': 'general'},
-                            'expected_output': {'answer': '4', 'confidence': 1.0},
-                            'output': {'answer': '4', 'confidence': 1.0},
-                            'metrics': {},
-                            'attributes': {},
-                            'scores': {
-                                'confidence': {
-                                    'name': 'confidence',
-                                    'value': 1.0,
-                                    'reason': None,
-                                    'source': {'name': 'SimpleEvaluator', 'arguments': None},
-                                }
-                            },
-                            'labels': {},
-                            'assertions': {
-                                'correct': {
-                                    'name': 'correct',
-                                    'value': True,
-                                    'reason': None,
-                                    'source': {'name': 'SimpleEvaluator', 'arguments': None},
-                                }
-                            },
-                            'task_duration': 1.0,
-                            'total_duration': 10.0,
-                            'trace_id': '00000000000000000000000000000001',
-                            'span_id': '0000000000000003',
-                            'evaluator_failures': [],
-                        },
-                        {
-                            'name': 'case2',
-                            'inputs': {'query': 'What is the capital of France?'},
-                            'metadata': {'difficulty': 'medium', 'category': 'geography'},
-                            'expected_output': {'answer': 'Paris', 'confidence': 1.0},
-                            'output': {'answer': 'Paris', 'confidence': 1.0},
-                            'metrics': {},
-                            'attributes': {},
-                            'scores': {
-                                'confidence': {
-                                    'name': 'confidence',
-                                    'value': 1.0,
-                                    'reason': None,
-                                    'source': {'name': 'SimpleEvaluator', 'arguments': None},
-                                }
-                            },
-                            'labels': {},
-                            'assertions': {
-                                'correct': {
-                                    'name': 'correct',
-                                    'value': True,
-                                    'reason': None,
-                                    'source': {'name': 'SimpleEvaluator', 'arguments': None},
-                                }
-                            },
-                            'task_duration': 1.0,
-                            'total_duration': 8.0,
-                            'trace_id': '00000000000000000000000000000001',
-                            'span_id': '0000000000000007',
-                            'evaluator_failures': [],
-                        },
-                    ],
-                    'failures': [],
-                    'averages': {
-                        'name': 'Averages',
-                        'scores': {'confidence': 1.0},
-                        'labels': {},
-                        'metrics': {},
-                        'assertions': 1.0,
-                        'task_duration': 1.0,
-                        'total_duration': 9.0,
-                    },
                     'logfire.json_schema': {
                         'type': 'object',
-                        'properties': {
-                            'name': {},
-                            'cases': {'type': 'array'},
-                            'failures': {'type': 'array'},
-                            'averages': {'type': 'object'},
-                        },
+                        'properties': {'name': {}, 'n_cases': {}, 'assertion_pass_rate': {}},
                     },
                 },
             ),