update message history validation to be per message

guardrails/run/runner.py

@@ -344,25 +344,42 @@ def validate_msg_history(
         )
         iteration = Iteration(call_id=call_log.id, index=attempt_number, inputs=inputs)
         call_log.iterations.insert(0, iteration)
-        value, _metadata = validator_service.validate(
-            value=msg_str,
-            metadata=self.metadata,
-            validator_map=self.validation_map,
-            iteration=iteration,
-            disable_tracer=self._disable_tracer,
-            path="msg_history",
-        )
-        validated_msg_history = validator_service.post_process_validation(
-            value, attempt_number, iteration, OutputTypes.STRING
-        )
 
-        iteration.outputs.validation_response = validated_msg_history
-        if isinstance(validated_msg_history, ReAsk):
-            raise ValidationError(
-                f"Message history validation failed: " f"{validated_msg_history}"
+        validated_msgs = ""
+
+        for msg in msg_history:
+            content = (
+                msg["content"].source
+                if isinstance(msg["content"], Prompt)
+                else msg["content"]
             )
-        if validated_msg_history != msg_str:
-            raise ValidationError("Message history validation failed")
+
+            value, _metadata = validator_service.validate(
+                value=content,
+                metadata=self.metadata,
+                validator_map=self.validation_map,
+                iteration=iteration,
+                disable_tracer=self._disable_tracer,
+                path="msg_history",
+            )
+
+            validated_msg = validator_service.post_process_validation(
+                value, attempt_number, iteration, OutputTypes.STRING
+            )
+
+            if isinstance(validated_msg, ReAsk):
+                raise ValidationError(
+                    f"Message history validation failed: {validated_msg}"
+                )
+            elif not validated_msg or iteration.status == fail_status:
+                raise ValidationError("Message history validation failed")
+
+            msg["content"] = cast(str, validated_msg)
+            validated_msgs += validated_msg
+
+        iteration.outputs.validation_response = validated_msgs
+
+        return msg_history  # type: ignore
 
     def prepare_msg_history(
         self,
@@ -380,7 +397,9 @@ def prepare_msg_history(
 
         # validate msg_history
         if "msg_history" in self.validation_map:
-            self.validate_msg_history(call_log, formatted_msg_history, attempt_number)
+            formatted_msg_history = self.validate_msg_history(  # type: ignore
+                call_log, formatted_msg_history, attempt_number
+            )
 
         return formatted_msg_history
 

tests/unit_tests/test_validator_base.py

@@ -313,8 +313,10 @@ class Pet(BaseModel):
 
 
 def test_input_validation_fix(mocker):
-    def mock_llm_api(*args, **kwargs):
-        return json.dumps({"name": "Fluffy"})
+    # Create a mock object for the mock_llm_api function
+    mock_llm_api = mocker.Mock()
+
+    mock_llm_api.return_value = json.dumps({"name": "Fluffy"})
 
     # fix returns an amended value for prompt/instructions validation,
     guard = Guard.from_pydantic(output_class=Pet)
@@ -340,23 +342,21 @@ def mock_llm_api(*args, **kwargs):
         == "But really,"
     )
 
-    # but raises for msg_history validation
     guard = Guard.from_pydantic(output_class=Pet)
     guard.use(TwoWords(on_fail=OnFailAction.FIX), on="msg_history")
 
-    with pytest.raises(ValidationError) as excinfo:
-        guard(
-            mock_llm_api,
-            msg_history=[
-                {
-                    "role": "user",
-                    "content": "What kind of pet should I get?",
-                }
-            ],
-        )
-    assert str(excinfo.value) == "Message history validation failed"
-    assert isinstance(guard.history.first.exception, ValidationError)
-    assert guard.history.first.exception == excinfo.value
+    guard(
+        mock_llm_api,
+        msg_history=[
+            {
+                "role": "user",
+                "content": "What kind of pet should I get?",
+            }
+        ],
+    )
+    assert mock_llm_api.call_count == 3
+    # last call should have "What kind" as the input
+    assert mock_llm_api.call_args.kwargs["msg_history"][0]["content"] == "What kind"
 
     # rail prompt validation
     guard = Guard.from_rail_string(