Fix some issues with non-serializable inputs in evals (#1333)

dmontagu · web-flow · commit ebd86fad5e7a · 2025-04-01T22:46:37.000Z
diff --git a/pydantic_evals/pydantic_evals/dataset.py b/pydantic_evals/pydantic_evals/dataset.py
@@ -26,7 +26,7 @@
 import yaml
 from pydantic import BaseModel, ConfigDict, Field, TypeAdapter, ValidationError, model_serializer
 from pydantic._internal import _typing_extra
-from pydantic_core import to_json, to_jsonable_python
+from pydantic_core import to_json
 from pydantic_core.core_schema import SerializationInfo, SerializerFunctionWrapHandler
 from typing_extensions import NotRequired, Self, TypedDict, TypeVar
 
@@ -907,11 +907,9 @@ async def _run_task_and_evaluators(
             span_id = f'{context.span_id:016x}'
         fallback_duration = time.time() - t0
 
-    report_inputs = to_jsonable_python(case.inputs)
-
     return ReportCase(
         name=report_case_name,
-        inputs=report_inputs,
+        inputs=case.inputs,
         metadata=case.metadata,
         expected_output=case.expected_output,
         output=scoring_context.output,
diff --git a/pydantic_evals/pydantic_evals/evaluators/llm_as_a_judge.py b/pydantic_evals/pydantic_evals/evaluators/llm_as_a_judge.py
@@ -83,6 +83,8 @@ def _stringify(value: Any) -> str:
     if isinstance(value, str):
         return value
     try:
+        # If the value can be serialized to JSON, use that.
+        # If that behavior is undesirable, the user could manually call repr on the arguments to the judge_* functions
         return to_json(value).decode()
     except Exception:
         return repr(value)
diff --git a/pydantic_evals/pydantic_evals/reporting/__init__.py b/pydantic_evals/pydantic_evals/reporting/__init__.py
@@ -158,12 +158,15 @@ def print(
         width: int | None = None,
         baseline: EvaluationReport | None = None,
         include_input: bool = False,
+        include_metadata: bool = False,
+        include_expected_output: bool = False,
         include_output: bool = False,
         include_durations: bool = True,
         include_total_duration: bool = False,
         include_removed_cases: bool = False,
         include_averages: bool = True,
         input_config: RenderValueConfig | None = None,
+        metadata_config: RenderValueConfig | None = None,
         output_config: RenderValueConfig | None = None,
         score_configs: dict[str, RenderNumberConfig] | None = None,
         label_configs: dict[str, RenderValueConfig] | None = None,
@@ -177,12 +180,15 @@ def print(
         table = self.console_table(
             baseline=baseline,
             include_input=include_input,
+            include_metadata=include_metadata,
+            include_expected_output=include_expected_output,
             include_output=include_output,
             include_durations=include_durations,
             include_total_duration=include_total_duration,
             include_removed_cases=include_removed_cases,
             include_averages=include_averages,
             input_config=input_config,
+            metadata_config=metadata_config,
             output_config=output_config,
             score_configs=score_configs,
             label_configs=label_configs,
@@ -195,12 +201,15 @@ def console_table(
         self,
         baseline: EvaluationReport | None = None,
         include_input: bool = False,
+        include_metadata: bool = False,
+        include_expected_output: bool = False,
         include_output: bool = False,
         include_durations: bool = True,
         include_total_duration: bool = False,
         include_removed_cases: bool = False,
         include_averages: bool = True,
         input_config: RenderValueConfig | None = None,
+        metadata_config: RenderValueConfig | None = None,
         output_config: RenderValueConfig | None = None,
         score_configs: dict[str, RenderNumberConfig] | None = None,
         label_configs: dict[str, RenderValueConfig] | None = None,
@@ -213,12 +222,15 @@ def console_table(
         """
         renderer = EvaluationRenderer(
             include_input=include_input,
+            include_metadata=include_metadata,
+            include_expected_output=include_expected_output,
             include_output=include_output,
             include_durations=include_durations,
             include_total_duration=include_total_duration,
             include_removed_cases=include_removed_cases,
             include_averages=include_averages,
             input_config={**_DEFAULT_VALUE_CONFIG, **(input_config or {})},
+            metadata_config={**_DEFAULT_VALUE_CONFIG, **(metadata_config or {})},
             output_config=output_config or _DEFAULT_VALUE_CONFIG,
             score_configs=score_configs or {},
             label_configs=label_configs or {},
@@ -496,6 +508,8 @@ def render_diff(self, name: str | None, old: T_contra | None, new: T_contra | No
 @dataclass
 class ReportCaseRenderer:
     include_input: bool
+    include_metadata: bool
+    include_expected_output: bool
     include_output: bool
     include_scores: bool
     include_labels: bool
@@ -505,6 +519,7 @@ class ReportCaseRenderer:
     include_total_duration: bool
 
     input_renderer: _ValueRenderer
+    metadata_renderer: _ValueRenderer
     output_renderer: _ValueRenderer
     score_renderers: dict[str, _NumberRenderer]
     label_renderers: dict[str, _ValueRenderer]
@@ -517,6 +532,10 @@ def build_base_table(self, title: str) -> Table:
         table.add_column('Case ID', style='bold')
         if self.include_input:
             table.add_column('Inputs', overflow='fold')
+        if self.include_metadata:
+            table.add_column('Metadata', overflow='fold')
+        if self.include_expected_output:
+            table.add_column('Expected Output', overflow='fold')
         if self.include_output:
             table.add_column('Outputs', overflow='fold')
         if self.include_scores:
@@ -538,6 +557,12 @@ def build_row(self, case: ReportCase) -> list[str]:
         if self.include_input:
             row.append(self.input_renderer.render_value(None, case.inputs) or EMPTY_CELL_STR)
 
+        if self.include_metadata:
+            row.append(self.input_renderer.render_value(None, case.metadata) or EMPTY_CELL_STR)
+
+        if self.include_expected_output:
+            row.append(self.input_renderer.render_value(None, case.expected_output) or EMPTY_CELL_STR)
+
         if self.include_output:
             row.append(self.output_renderer.render_value(None, case.output) or EMPTY_CELL_STR)
 
@@ -565,6 +590,12 @@ def build_aggregate_row(self, aggregate: ReportCaseAggregate) -> list[str]:
         if self.include_input:
             row.append(EMPTY_AGGREGATE_CELL_STR)
 
+        if self.include_metadata:
+            row.append(EMPTY_AGGREGATE_CELL_STR)
+
+        if self.include_expected_output:
+            row.append(EMPTY_AGGREGATE_CELL_STR)
+
         if self.include_output:
             row.append(EMPTY_AGGREGATE_CELL_STR)
 
@@ -598,6 +629,19 @@ def build_diff_row(
             input_diff = self.input_renderer.render_diff(None, baseline.inputs, new_case.inputs) or EMPTY_CELL_STR
             row.append(input_diff)
 
+        if self.include_metadata:
+            metadata_diff = (
+                self.metadata_renderer.render_diff(None, baseline.metadata, new_case.metadata) or EMPTY_CELL_STR
+            )
+            row.append(metadata_diff)
+
+        if self.include_expected_output:
+            expected_output_diff = (
+                self.output_renderer.render_diff(None, baseline.expected_output, new_case.expected_output)
+                or EMPTY_CELL_STR
+            )
+            row.append(expected_output_diff)
+
         if self.include_output:
             output_diff = self.output_renderer.render_diff(None, baseline.output, new_case.output) or EMPTY_CELL_STR
             row.append(output_diff)
@@ -642,6 +686,12 @@ def build_diff_aggregate_row(
         if self.include_input:
             row.append(EMPTY_AGGREGATE_CELL_STR)
 
+        if self.include_metadata:
+            row.append(EMPTY_AGGREGATE_CELL_STR)
+
+        if self.include_expected_output:
+            row.append(EMPTY_AGGREGATE_CELL_STR)
+
         if self.include_output:
             row.append(EMPTY_AGGREGATE_CELL_STR)
 
@@ -777,6 +827,8 @@ class EvaluationRenderer:
 
     # Columns to include
     include_input: bool
+    include_metadata: bool
+    include_expected_output: bool
     include_output: bool
     include_durations: bool
     include_total_duration: bool
@@ -786,6 +838,7 @@ class EvaluationRenderer:
     include_averages: bool
 
     input_config: RenderValueConfig
+    metadata_config: RenderValueConfig
     output_config: RenderValueConfig
     score_configs: dict[str, RenderNumberConfig]
     label_configs: dict[str, RenderValueConfig]
@@ -820,6 +873,7 @@ def _get_case_renderer(
         self, report: EvaluationReport, baseline: EvaluationReport | None = None
     ) -> ReportCaseRenderer:
         input_renderer = _ValueRenderer.from_config(self.input_config)
+        metadata_renderer = _ValueRenderer.from_config(self.metadata_config)
         output_renderer = _ValueRenderer.from_config(self.output_config)
         score_renderers = self._infer_score_renderers(report, baseline)
         label_renderers = self._infer_label_renderers(report, baseline)
@@ -830,6 +884,8 @@ def _get_case_renderer(
 
         return ReportCaseRenderer(
             include_input=self.include_input,
+            include_metadata=self.include_metadata,
+            include_expected_output=self.include_expected_output,
             include_output=self.include_output,
             include_scores=self.include_scores(report, baseline),
             include_labels=self.include_labels(report, baseline),
@@ -838,6 +894,7 @@ def _get_case_renderer(
             include_durations=self.include_durations,
             include_total_duration=self.include_total_duration,
             input_renderer=input_renderer,
+            metadata_renderer=metadata_renderer,
             output_renderer=output_renderer,
             score_renderers=score_renderers,
             label_renderers=label_renderers,
diff --git a/tests/evals/test_dataset.py b/tests/evals/test_dataset.py
@@ -13,6 +13,7 @@
 from pydantic import BaseModel
 
 from ..conftest import try_import
+from .utils import render_table
 
 with try_import() as imports_successful:
     from pydantic_evals import Case, Dataset
@@ -342,40 +343,42 @@ async def my_task(inputs: TaskInput) -> TaskOutput:
         return TaskOutput(answer=f'answer to {inputs.query}')
 
     report = await example_dataset.evaluate(my_task)
-    assert report.cases == [
-        ReportCase(
-            name='case1',
-            inputs={'query': 'What is 2+2?'},
-            metadata=TaskMetadata(difficulty='easy', category='general'),
-            expected_output=TaskOutput(answer='4', confidence=1.0),
-            output=TaskOutput(answer='answer to What is 2+2?', confidence=1.0),
-            metrics={'chars': 12},
-            attributes={'is_about_france': False},
-            scores={},
-            labels={},
-            assertions={},
-            task_duration=1.0,
-            total_duration=3.0,
-            trace_id='00000000000000000000000000000001',
-            span_id='0000000000000003',
-        ),
-        ReportCase(
-            name='case2',
-            inputs={'query': 'What is the capital of France?'},
-            metadata=TaskMetadata(difficulty='medium', category='geography'),
-            expected_output=TaskOutput(answer='Paris', confidence=1.0),
-            output=TaskOutput(answer='answer to What is the capital of France?', confidence=1.0),
-            metrics={'chars': 30},
-            attributes={'is_about_france': True},
-            scores={},
-            labels={},
-            assertions={},
-            task_duration=1.0,
-            total_duration=3.0,
-            trace_id='00000000000000000000000000000001',
-            span_id='0000000000000007',
-        ),
-    ]
+    assert report.cases == snapshot(
+        [
+            ReportCase(
+                name='case1',
+                inputs=TaskInput(query='What is 2+2?'),
+                metadata=TaskMetadata(difficulty='easy', category='general'),
+                expected_output=TaskOutput(answer='4', confidence=1.0),
+                output=TaskOutput(answer='answer to What is 2+2?', confidence=1.0),
+                metrics={'chars': 12},
+                attributes={'is_about_france': False},
+                scores={},
+                labels={},
+                assertions={},
+                task_duration=1.0,
+                total_duration=3.0,
+                trace_id='00000000000000000000000000000001',
+                span_id='0000000000000003',
+            ),
+            ReportCase(
+                name='case2',
+                inputs=TaskInput(query='What is the capital of France?'),
+                metadata=TaskMetadata(difficulty='medium', category='geography'),
+                expected_output=TaskOutput(answer='Paris', confidence=1.0),
+                output=TaskOutput(answer='answer to What is the capital of France?', confidence=1.0),
+                metrics={'chars': 30},
+                attributes={'is_about_france': True},
+                scores={},
+                labels={},
+                assertions={},
+                task_duration=1.0,
+                total_duration=3.0,
+                trace_id='00000000000000000000000000000001',
+                span_id='0000000000000007',
+            ),
+        ]
+    )
 
 
 async def test_repeated_name_outputs(example_dataset: Dataset[TaskInput, TaskOutput, TaskMetadata]):
@@ -393,7 +396,7 @@ async def my_task(inputs: TaskInput) -> TaskOutput:
         [
             ReportCase(
                 name='case1',
-                inputs={'query': 'What is 2+2?'},
+                inputs=TaskInput(query='What is 2+2?'),
                 metadata=TaskMetadata(difficulty='easy', category='general'),
                 expected_output=TaskOutput(answer='4', confidence=1.0),
                 output=TaskOutput(answer='answer to What is 2+2?', confidence=1.0),
@@ -419,7 +422,7 @@ async def my_task(inputs: TaskInput) -> TaskOutput:
             ),
             ReportCase(
                 name='case2',
-                inputs={'query': 'What is the capital of France?'},
+                inputs=TaskInput(query='What is the capital of France?'),
                 metadata=TaskMetadata(difficulty='medium', category='geography'),
                 expected_output=TaskOutput(answer='Paris', confidence=1.0),
                 output=TaskOutput(answer='answer to What is the capital of France?', confidence=1.0),
@@ -467,7 +470,7 @@ async def my_task(inputs: TaskInput) -> TaskOutput:
         [
             ReportCase(
                 name='case1',
-                inputs={'query': 'What is 2+2?'},
+                inputs=TaskInput(query='What is 2+2?'),
                 metadata=TaskMetadata(difficulty='easy', category='general'),
                 expected_output=TaskOutput(answer='4', confidence=1.0),
                 output=TaskOutput(answer='answer to What is 2+2?', confidence=1.0),
@@ -483,7 +486,7 @@ async def my_task(inputs: TaskInput) -> TaskOutput:
             ),
             ReportCase(
                 name='case2',
-                inputs={'query': 'What is the capital of France?'},
+                inputs=TaskInput(query='What is the capital of France?'),
                 metadata=TaskMetadata(difficulty='medium', category='geography'),
                 expected_output=TaskOutput(answer='Paris', confidence=1.0),
                 output=TaskOutput(answer='answer to What is the capital of France?', confidence=1.0),
@@ -988,3 +991,47 @@ def test_import_generate_dataset():
     from pydantic_evals.generation import generate_dataset
 
     assert generate_dataset
+
+
+def test_evaluate_non_serializable_inputs():
+    @dataclass
+    class MyInputs:
+        result_type: type[str] | type[int]
+
+    my_dataset = Dataset[MyInputs, Any, Any](
+        cases=[
+            Case(
+                name='str',
+                inputs=MyInputs(result_type=str),
+                expected_output='abc',
+            ),
+            Case(
+                name='int',
+                inputs=MyInputs(result_type=int),
+                expected_output=123,
+            ),
+        ],
+    )
+
+    async def my_task(my_inputs: MyInputs) -> int | str:
+        if issubclass(my_inputs.result_type, str):
+            return my_inputs.result_type('abc')
+        else:
+            return my_inputs.result_type(123)
+
+    report = my_dataset.evaluate_sync(task=my_task)
+    assert [c.inputs for c in report.cases] == snapshot([MyInputs(result_type=str), MyInputs(result_type=int)])
+
+    table = report.console_table(include_input=True)
+    assert render_table(table) == snapshot("""\
+                                        Evaluation Summary: my_task
+┏━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┓
+┃ Case ID  ┃ Inputs                                                                             ┃ Duration ┃
+┡━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━┩
+│ str      │ test_evaluate_non_serializable_inputs.<locals>.MyInputs(result_type=<class 'str'>) │     1.0s │
+├──────────┼────────────────────────────────────────────────────────────────────────────────────┼──────────┤
+│ int      │ test_evaluate_non_serializable_inputs.<locals>.MyInputs(result_type=<class 'int'>) │     1.0s │
+├──────────┼────────────────────────────────────────────────────────────────────────────────────┼──────────┤
+│ Averages │                                                                                    │     1.0s │
+└──────────┴────────────────────────────────────────────────────────────────────────────────────┴──────────┘
+""")
diff --git a/tests/evals/test_reporting.py b/tests/evals/test_reporting.py
diff --git a/tests/evals/utils.py b/tests/evals/utils.py
