Bug/onfail enum match (#701)

* fix on_fail enum matching

* lint fix

* fix noop behavior

* fix async old openai tests

* lint

Author: zsimjee

guardrails/classes/history/call.py

@@ -269,10 +269,28 @@ def guarded_output(self) -> Optional[Union[str, Dict]]:
         validated output may be "fixed" values that were corrected
         during validation.
 
-        This will only have a value if the Guard is in a passing state.
+        This will only have a value if the Guard is in a passing state
+        OR if the action is no-op.
         """
         if self.status == pass_status:
             return self.fixed_output
+        last_iteration = self.iterations.last
+        if (
+            not self.status == pass_status
+            and last_iteration
+            and last_iteration.failed_validations
+        ):
+            # check that all failed validations are noop or none
+            all_noop = True
+            for failed_validation in last_iteration.failed_validations:
+                if (
+                    failed_validation.value_after_validation
+                    is not failed_validation.value_before_validation
+                ):
+                    all_noop = False
+                    break
+            if all_noop:
+                return last_iteration.guarded_output
 
     @property
     @deprecated(

guardrails/validator_base.py

@@ -422,14 +422,23 @@ def __init__(
                     """,
                     FutureWarning,
                 )
+        self.on_fail_descriptor: Union[str, OnFailAction] = "custom"
 
         if on_fail is None:
             on_fail = OnFailAction.NOOP
         if isinstance(on_fail, OnFailAction):
             self.on_fail_descriptor = on_fail
             self.on_fail_method = None
+        elif (
+            isinstance(on_fail, str)
+            and OnFailAction.__members__.get(on_fail.upper()) is not None
+        ):
+            self.on_fail_descriptor = (
+                OnFailAction.__members__.get(on_fail.upper())
+                or ""  # this default isn't needed, it's just for pyright
+            )
+            self.on_fail_method = None
         else:
-            self.on_fail_descriptor = "custom"
             self.on_fail_method = on_fail
 
         # Store the kwargs for the validator.

tests/integration_tests/test_async.py

@@ -97,7 +97,9 @@ async def test_entity_extraction_with_noop(mocker):
     # assert final_output.validated_output == entity_extraction.VALIDATED_OUTPUT_NOOP
 
     assert final_output.validation_passed is False
-    assert final_output.validated_output is None
+    assert final_output.validated_output is not None
+    assert final_output.validated_output["fees"]
+    assert final_output.validated_output["interest_rates"]
 
     call = guard.history.first
 
@@ -131,7 +133,9 @@ async def test_entity_extraction_with_noop_pydantic(mocker):
 
     # Assertions are made on the guard state object.
     assert final_output.validation_passed is False
-    assert final_output.validated_output is None
+    assert final_output.validated_output is not None
+    assert final_output.validated_output["fees"]
+    assert final_output.validated_output["interest_rates"]
 
     call = guard.history.first
 

tests/integration_tests/test_guard.py

@@ -238,7 +238,11 @@ def test_entity_extraction_with_noop(mocker, rail, prompt):
 
     # Assertions are made on the guard state object.
     assert final_output.validation_passed is False
-    assert final_output.validated_output is None
+    assert (
+        final_output.validated_output is not None
+        and validated_output.__get__("fees")
+        and validated_output.__get__("interest_rates")
+    )
 
     call = guard.history.first
 
@@ -248,7 +252,11 @@ def test_entity_extraction_with_noop(mocker, rail, prompt):
     # For orginal prompt and output
     assert call.compiled_prompt == entity_extraction.COMPILED_PROMPT
     assert call.raw_outputs.last == entity_extraction.LLM_OUTPUT
-    assert call.guarded_output is None
+    assert (
+        call.guarded_output is not None
+        and call.guarded_output["fees"]
+        and call.guarded_output["interest_rates"]
+    )
     assert call.validation_response == entity_extraction.VALIDATED_OUTPUT_NOOP
 
 
@@ -843,8 +851,10 @@ def invoke(
     model = MockModel()
     guard = (
         Guard()
-        .use(RegexMatch("Ice cream", match_type="search"), on="output")
-        .use(ReadingTime(0.05))  # 3 seconds
+        .use(
+            RegexMatch("Ice cream", match_type="search", on_fail="refrain"), on="output"
+        )
+        .use(ReadingTime(0.05, on_fail="refrain"))  # 3 seconds
     )
     output_parser = StrOutputParser()
 

tests/integration_tests/test_multi_reask.py

@@ -43,5 +43,5 @@ def test_multi_reask(mocker):
     # The output here fails some validators but passes others.
     # Since those that it fails in the end are noop fixes, validation fails.
     assert call.validation_response == python_rail.VALIDATOR_PARALLELISM_RESPONSE_3
-    assert call.guarded_output is None
+    assert call.guarded_output is not None and isinstance(call.guarded_output, str)
     assert call.status == "fail"

tests/integration_tests/test_validator_base.py

@@ -0,0 +1,58 @@
+from typing import Any, Dict
+
+from guardrails.guard import Guard
+from guardrails.validator_base import (
+    FailResult,
+    ValidationResult,
+    Validator,
+    register_validator,
+)
+
+
+@register_validator("failure", "string")
+class FailureValidator(Validator):
+    def validate(self, value: Any, metadata: Dict[str, Any]) -> ValidationResult:
+        return FailResult(
+            error_message=("Failed cuz this is the failure validator"),
+            fix_value="FIXED",
+        )
+
+
+# TODO: Add reask tests. Reask is fairly well covered through notebooks
+# but it's good to have it here too.
+def test_fix():
+    guard = Guard().use(FailureValidator, on_fail="fix")
+    res = guard.parse("hi")
+    assert res.validated_output == "FIXED"
+    assert res.validation_passed  # Should this even be true though?
+
+
+def test_default_noop():
+    guard = Guard().use(FailureValidator, on_fail="noop")
+    res = guard.parse("hi")
+    assert res.validated_output == "hi"
+    assert not res.validation_passed
+
+
+def test_filter():
+    guard = Guard().use(FailureValidator, on_fail="filter")
+    res = guard.parse("hi")
+    assert res.validated_output is None
+    assert not res.validation_passed
+
+
+def test_refrain():
+    guard = Guard().use(FailureValidator, on_fail="refrain")
+    res = guard.parse("hi")
+    assert res.validated_output is None
+    assert not res.validation_passed
+
+
+def test_exception():
+    guard = Guard().use(FailureValidator, on_fail="exception")
+    try:
+        guard.parse("hi")
+    except Exception as e:
+        assert "Failed cuz this is the failure validator" in str(e)
+    else:
+        assert False, "Expected an exception"