wip: validation summary integration

Author: AlejandroEsquivel

guardrails/classes/validation/validation_summary.py

@@ -0,0 +1,35 @@
+# TODO Temp to update once generated class is in
+from typing import List, Optional
+
+from guardrails.classes.generic.arbitrary_model import ArbitraryModel
+from guardrails.classes.validation.validation_result import ErrorSpan, FailResult
+from guardrails.classes.validation.validator_logs import ValidatorLogs
+
+
+class ValidationSummary(ArbitraryModel):
+    validator_name: str
+    validator_status: str
+    failure_reason: Optional[str]
+    error_spans: Optional[List["ErrorSpan"]] = []
+    property_path: Optional[str]
+
+    @staticmethod
+    def from_validator_logs(
+        validator_logs: List[ValidatorLogs],
+    ) -> List["ValidationSummary"]:
+        summaries = []
+        for log in validator_logs:
+            validation_result = log.validation_result
+            is_fail_result = isinstance(validation_result, FailResult)
+            failure_reason = validation_result.error_message if is_fail_result else None
+            error_spans = validation_result.error_spans if is_fail_result else []
+            summaries.append(
+                ValidationSummary(
+                    validator_name=log.validator_name,
+                    validator_status=log.validation_result.outcome,
+                    property_path=log.property_path,
+                    failure_reason=failure_reason,
+                    error_spans=error_spans,
+                )
+            )
+        return summaries

guardrails/classes/validation_outcome.py

@@ -1,4 +1,4 @@
-from typing import Generic, Iterator, Optional, Tuple, Union, cast
+from typing import Generic, Iterator, List, Optional, Tuple, Union, cast
 
 from pydantic import Field
 from rich.pretty import pretty_repr
@@ -11,6 +11,7 @@
 from guardrails.classes.history import Call, Iteration
 from guardrails.classes.output_type import OT
 from guardrails.classes.generic.arbitrary_model import ArbitraryModel
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.constants import pass_status
 from guardrails.utils.safe_get import safe_get
 
@@ -31,6 +32,11 @@ class ValidationOutcome(IValidationOutcome, ArbitraryModel, Generic[OT]):
         error: If the validation failed, this field will contain the error message
     """
 
+    validation_summaries: Optional[List["ValidationSummary"]] = Field(
+        description="The summaries of the validation results.", default=[]
+    )
+    """The summaries of the validation results."""
+
     raw_llm_output: Optional[str] = Field(
         description="The raw, unchanged output from the LLM call.", default=None
     )
@@ -75,6 +81,8 @@ def from_guard_history(cls, call: Call):
             list(last_iteration.reasks), 0
         )
         validation_passed = call.status == pass_status
+        validator_logs = last_iteration.validator_logs or []
+        validation_summaries = ValidationSummary.from_validator_logs(validator_logs)
         reask = last_output if isinstance(last_output, ReAsk) else None
         error = call.error
         output = cast(OT, call.guarded_output)
@@ -84,6 +92,7 @@ def from_guard_history(cls, call: Call):
             validated_output=output,
             reask=reask,
             validation_passed=validation_passed,
+            validation_summaries=validation_summaries,
             error=error,
         )
 

guardrails/guard.py

@@ -1193,6 +1193,11 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O
             )
             self.history.extend([Call.from_interface(call) for call in guard_history])
 
+            # TODO Validation Summary
+            # validator_logs = self.history.last.iterations.last.validator_logs
+            # validation_summaries = ValidationSummary.
+            # from_validator_logs(validator_logs)
+
             # TODO: See if the below statement is still true
             # Our interfaces are too different for this to work right now.
             # Once we move towards shared interfaces for both the open source
@@ -1203,6 +1208,7 @@ def _single_server_call(self, *, payload: Dict[str, Any]) -> ValidationOutcome[O
                 if validation_output.validated_output
                 else None
             )
+            # TODO: Validation Summary
             return ValidationOutcome[OT](
                 call_id=validation_output.call_id,  # type: ignore
                 raw_llm_output=validation_output.raw_llm_output,
@@ -1224,9 +1230,11 @@ def _stream_server_call(
                 payload=ValidatePayload.from_dict(payload),  # type: ignore
                 openai_api_key=get_call_kwarg("api_key"),
             )
+            print("Server response:", response)
             for fragment in response:
                 validation_output = fragment
                 if validation_output is None:
+                    # TODO Validation Summary
                     yield ValidationOutcome[OT](
                         call_id="0",  # type: ignore
                         raw_llm_output=None,
@@ -1240,6 +1248,7 @@ def _stream_server_call(
                         if validation_output.validated_output
                         else None
                     )
+                    # TODO Validation Summary
                     yield ValidationOutcome[OT](
                         call_id=validation_output.call_id,  # type: ignore
                         raw_llm_output=validation_output.raw_llm_output,

guardrails/run/async_stream_runner.py

@@ -13,6 +13,7 @@
 from guardrails.classes import ValidationOutcome
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OutputTypes
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.constants import pass_status
 from guardrails.llm_providers import (
     AsyncLiteLLMCallable,
@@ -164,11 +165,16 @@ async def async_step(
                     )
                 validation_response += cast(str, validated_fragment)
                 passed = call_log.status == pass_status
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=chunk_text,
                     validated_output=validated_fragment,
                     validation_passed=passed,
+                    validation_summaries=validation_summaries,
                 )
         else:
             async for chunk in stream_output:
@@ -204,11 +210,17 @@ async def async_step(
                     validation_response = cast(list, validated_fragment)
                 else:
                     validation_response = cast(dict, validated_fragment)
+
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=chunk_text,
                     validation_passed=validated_fragment is not None,
+                    validation_summaries=validation_summaries,
                 )
 
         iteration.outputs.raw_output = fragment

guardrails/run/stream_runner.py

@@ -1,8 +1,10 @@
 from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union, cast
 
+
 from guardrails import validator_service
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OT, OutputTypes
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.llm_providers import (
     LiteLLMCallable,
@@ -176,7 +178,9 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 "$",
                 validate_subschema=True,
             )
-
+            # Not sure I like adding all this info to every chunk
+            # maybe move last chunk?
+            validator_logs = iteration.validator_logs
             for res in gen:
                 chunk = res.chunk
                 original_text = res.original_text
@@ -195,13 +199,19 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                     )
                 # 5. Convert validated fragment to a pretty JSON string
                 validation_response += cast(str, chunk)
+                validator_logs = call_log.iterations.last.validator_logs
+
+                validation_summaries = ValidationSummary.from_validator_logs(
+                    validator_logs
+                )
                 passed = call_log.status == pass_status
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     #  The chunk or the whole output?
                     raw_llm_output=original_text,
                     validated_output=chunk,
                     validation_passed=passed,
+                    validation_summaries=validation_summaries,
                 )
 
         # handle non string schema
@@ -246,11 +256,17 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 else:
                     validation_response = cast(dict, validated_fragment)
                 # 5. Convert validated fragment to a pretty JSON string
+
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=validated_fragment,
                     validation_passed=validated_fragment is not None,
+                    validation_summaries=validation_summaries,
                 )
 
         # # Finally, add to logs