changed to only output validation summaries with failures

Author: AlejandroEsquivel

guardrails/classes/validation/validation_summary.py

@@ -40,3 +40,15 @@ def from_validator_logs(
         ):
             summaries.append(summary)
         return summaries
+
+    @staticmethod
+    def from_validator_logs_only_fails(
+        validator_logs: List[ValidatorLogs],
+    ) -> List["ValidationSummary"]:
+        summaries = []
+        for summary in ValidationSummary._generate_summaries_from_validator_logs(
+            validator_logs
+        ):
+            if summary.failure_reason:
+                summaries.append(summary)
+        return summaries

guardrails/classes/validation_outcome.py

@@ -82,7 +82,9 @@ def from_guard_history(cls, call: Call):
         )
         validation_passed = call.status == pass_status
         validator_logs = last_iteration.validator_logs or []
-        validation_summaries = ValidationSummary.from_validator_logs(validator_logs)
+        validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+            validator_logs
+        )
         reask = last_output if isinstance(last_output, ReAsk) else None
         error = call.error
         output = cast(OT, call.guarded_output)

guardrails/run/async_stream_runner.py

@@ -13,6 +13,7 @@
 from guardrails.classes import ValidationOutcome
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OutputTypes
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.constants import pass_status
 from guardrails.llm_providers import (
     AsyncLiteLLMCallable,
@@ -164,11 +165,16 @@ async def async_step(
                     )
                 validation_response += cast(str, validated_fragment)
                 passed = call_log.status == pass_status
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=chunk_text,
                     validated_output=validated_fragment,
                     validation_passed=passed,
+                    validation_summaries=validation_summaries,
                 )
         else:
             async for chunk in stream_output:
@@ -204,11 +210,16 @@ async def async_step(
                     validation_response = cast(list, validated_fragment)
                 else:
                     validation_response = cast(dict, validated_fragment)
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=chunk_text,
                     validation_passed=validated_fragment is not None,
+                    validation_summaries=validation_summaries,
                 )
 
         iteration.outputs.raw_output = fragment

guardrails/run/stream_runner.py

@@ -1,8 +1,10 @@
 from typing import Any, Dict, Generator, Iterable, List, Optional, Tuple, Union, cast
 
+
 from guardrails import validator_service
 from guardrails.classes.history import Call, Inputs, Iteration, Outputs
 from guardrails.classes.output_type import OT, OutputTypes
+from guardrails.classes.validation.validation_summary import ValidationSummary
 from guardrails.classes.validation_outcome import ValidationOutcome
 from guardrails.llm_providers import (
     LiteLLMCallable,
@@ -176,8 +178,7 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 "$",
                 validate_subschema=True,
             )
-
-            for res in gen:
+            for chunk_index, res in enumerate(gen):
                 chunk = res.chunk
                 original_text = res.original_text
                 if isinstance(chunk, SkeletonReAsk):
@@ -196,12 +197,20 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 # 5. Convert validated fragment to a pretty JSON string
                 validation_response += cast(str, chunk)
                 passed = call_log.status == pass_status
+
+                validator_logs = call_log.iterations.last.validator_logs
+
+                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                    validator_logs
+                )
+
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     #  The chunk or the whole output?
                     raw_llm_output=original_text,
                     validated_output=chunk,
                     validation_passed=passed,
+                    validation_summaries=validation_summaries,
                 )
 
         # handle non string schema
@@ -246,11 +255,17 @@ def prepare_chunk_generator(stream) -> Iterable[Tuple[Any, bool]]:
                 else:
                     validation_response = cast(dict, validated_fragment)
                 # 5. Convert validated fragment to a pretty JSON string
+
+                validator_logs = iteration.validator_logs
+                validation_summaries = ValidationSummary.from_validator_logs_only_fails(
+                    validator_logs
+                )
                 yield ValidationOutcome(
                     call_id=call_log.id,  # type: ignore
                     raw_llm_output=fragment,
                     validated_output=validated_fragment,
                     validation_passed=validated_fragment is not None,
+                    validation_summaries=validation_summaries,
                 )
 
         # # Finally, add to logs