Include report averages (#3053)

Co-authored-by: Douwe Maan <[email protected]>

Author: cetra3

pydantic_evals/pydantic_evals/dataset.py

@@ -343,6 +343,8 @@ async def _handle_case(case: Case[InputsT, OutputT, MetadataT], report_case_name
                 trace_id=trace_id,
             )
             if (averages := report.averages()) is not None and averages.assertions is not None:
+                experiment_metadata = {'n_cases': len(self.cases), 'averages': averages}
+                eval_span.set_attribute('logfire.experiment.metadata', experiment_metadata)
                 eval_span.set_attribute('assertion_pass_rate', averages.assertions)
         return report
 

tests/evals/test_dataset.py

@@ -1555,6 +1555,16 @@ async def mock_async_task(inputs: TaskInput) -> TaskOutput:
                             'gen_ai.operation.name': {},
                             'n_cases': {},
                             'name': {},
+                            'logfire.experiment.metadata': {
+                                'type': 'object',
+                                'properties': {
+                                    'averages': {
+                                        'type': 'object',
+                                        'title': 'ReportCaseAggregate',
+                                        'x-python-datatype': 'PydanticModel',
+                                    }
+                                },
+                            },
                             'task_name': {},
                         },
                         'type': 'object',
@@ -1563,6 +1573,18 @@ async def mock_async_task(inputs: TaskInput) -> TaskOutput:
                     'logfire.msg_template': 'evaluate {name}',
                     'logfire.span_type': 'span',
                     'n_cases': 2,
+                    'logfire.experiment.metadata': {
+                        'n_cases': 2,
+                        'averages': {
+                            'name': 'Averages',
+                            'scores': {'confidence': 1.0},
+                            'labels': {},
+                            'metrics': {},
+                            'assertions': 1.0,
+                            'task_duration': 1.0,
+                            'total_duration': 9.0,
+                        },
+                    },
                     'name': 'mock_async_task',
                     'task_name': 'mock_async_task',
                 },