more progress

Author: dtam

guardrails/formatters/json_formatter.py

@@ -105,12 +105,16 @@ def fn(
                 messages: Optional[List[Dict[str, str]]] = None,
                 **kwargs,
             ) -> str:
+                prompt = ""
+                for msg in messages:
+                    prompt += msg["content"]
+
                 return json.dumps(
                     Jsonformer(
                         model=model.model,
                         tokenizer=model.tokenizer,
                         json_schema=self.output_schema,
-                        messages=messages,
+                        prompt=prompt
                     )()
                 )
 
@@ -127,12 +131,16 @@ def fn(
                 messages: Optional[List[Dict[str, str]]] = None,
                 **kwargs,
             ) -> str:
+                prompt = ""
+                for msg in messages:
+                    prompt += msg["content"]
+            
                 return json.dumps(
                     Jsonformer(
                         model=model,
                         tokenizer=tokenizer,
                         json_schema=self.output_schema,
-                        messages=messages,
+                        prompt=prompt
                     )()
                 )
 

guardrails/guard.py

@@ -647,8 +647,8 @@ def _execute(
             reask_messages=reask_messages,
         )
         metadata = metadata or {}
-        if not llm_output and llm_api and not (messages):
-            raise RuntimeError("'messages' must be provided in order to call an LLM!")
+        # if not llm_output and llm_api and not (messages):
+        #     raise RuntimeError("'messages' must be provided in order to call an LLM!")
 
         # check if validator requirements are fulfilled
         missing_keys = verify_metadata_requirements(metadata, self._validators)
@@ -841,11 +841,11 @@ def __call__(
         """
 
         messages = messages or self._exec_opts.messages or []
-        if messages is not None and not len(messages):
-            raise RuntimeError(
-                "You must provide messages. "
-                "Alternatively, you can provide a prompt in the Schema constructor."
-            )
+        # if messages is not None and not len(messages):
+        #     raise RuntimeError(
+        #         "You must provide messages. "
+        #         "Alternatively, you can provide a prompt in the Schema constructor."
+        #     )
         return trace_guard_execution(
             self.name,
             self.history,

guardrails/llm_providers.py

@@ -43,14 +43,14 @@ def nonchat_prompt(prompt: str, instructions: Optional[str] = None) -> str:
 def chat_prompt(
     prompt: Optional[str],
     instructions: Optional[str] = None,
-    msg_history: Optional[List[Dict]] = None,
+    messages: Optional[List[Dict]] = None,
 ) -> List[Dict[str, str]]:
     """Prepare final prompt for chat engine."""
-    if msg_history:
-        return msg_history
+    if messages:
+        return messages
     if prompt is None:
         raise PromptCallableException(
-            "You must pass in either `text` or `msg_history` to `guard.__call__`."
+            "You must pass in either `text` or `messages` to `guard.__call__`."
         )
 
     if not instructions:
@@ -65,14 +65,14 @@ def chat_prompt(
 def litellm_messages(
     prompt: Optional[str],
     instructions: Optional[str] = None,
-    msg_history: Optional[List[Dict]] = None,
+    messages: Optional[List[Dict]] = None,
 ) -> List[Dict[str, str]]:
     """Prepare messages for LiteLLM."""
-    if msg_history:
-        return msg_history
+    if messages:
+        return messages
     if prompt is None:
         raise PromptCallableException(
-            "Either `text` or `msg_history` required for `guard.__call__`."
+            "Either `text` or `messages` required for `guard.__call__`."
         )
 
     if instructions:
@@ -143,8 +143,7 @@ def _invoke_llm(
         self,
         text: Optional[str] = None,
         model: str = "gpt-3.5-turbo",
-        instructions: Optional[str] = None,
-        msg_history: Optional[List[Dict]] = None,
+        messages: Optional[List[Dict]] = None,
         *args,
         **kwargs,
     ) -> LLMResponse:
@@ -170,9 +169,9 @@ def _invoke_llm(
                 "The `litellm` package is not installed. "
                 "Install with `pip install litellm`"
             ) from e
-        if text is not None or instructions is not None or msg_history is not None:
+        if messages is not None:
             messages = litellm_messages(
-                prompt=text, instructions=instructions, msg_history=msg_history
+                prompt=text, messages=messages
             )
             kwargs["messages"] = messages
 
@@ -592,7 +591,7 @@ async def invoke_llm(
         self,
         text: Optional[str] = None,
         instructions: Optional[str] = None,
-        msg_history: Optional[List[Dict]] = None,
+        messages: Optional[List[Dict]] = None,
         *args,
         **kwargs,
     ):
@@ -619,11 +618,11 @@ async def invoke_llm(
                 "Install with `pip install litellm`"
             ) from e
 
-        if text is not None or instructions is not None or msg_history is not None:
+        if text is not None or instructions is not None or messages is not None:
             messages = litellm_messages(
                 prompt=text,
                 instructions=instructions,
-                msg_history=msg_history,
+                messages=messages,
             )
             kwargs["messages"] = messages
 

guardrails/run/async_runner.py

@@ -22,6 +22,8 @@
 from guardrails.actions.reask import NonParseableReAsk, ReAsk
 from guardrails.telemetry import trace_async_call, trace_async_step
 
+from guardrails.constants import fail_status
+from guardrails.prompt import Prompt
 
 class AsyncRunner(Runner):
     def __init__(
@@ -339,29 +341,37 @@ async def prepare_messages(
     async def validate_messages(
         self, call_log: Call, messages: MessageHistory, attempt_number: int
     ):
-        msg_str = messages_string(messages)
-        inputs = Inputs(
-            llm_output=msg_str,
-        )
-        iteration = Iteration(call_id=call_log.id, index=attempt_number, inputs=inputs)
-        call_log.iterations.insert(0, iteration)
-        value, _metadata = await validator_service.async_validate(
-            value=msg_str,
-            metadata=self.metadata,
-            validator_map=self.validation_map,
-            iteration=iteration,
-            disable_tracer=self._disable_tracer,
-            path="messages",
-        )
-        validated_messages = validator_service.post_process_validation(
-            value, attempt_number, iteration, OutputTypes.STRING
-        )
-        validated_messages = cast(str, validated_messages)
+        for msg in messages:
+            content = (
+                msg["content"].source
+                if isinstance(msg["content"], Prompt)
+                else msg["content"]
+            )
+            inputs = Inputs(
+                        llm_output=content,
+                    )
+            iteration = Iteration(call_id=call_log.id, index=attempt_number, inputs=inputs)
+            call_log.iterations.insert(0, iteration)
+            value, _metadata = await validator_service.async_validate(
+                value=content,
+                metadata=self.metadata,
+                validator_map=self.validation_map,
+                iteration=iteration,
+                disable_tracer=self._disable_tracer,
+                path="messages",
+            )
 
-        iteration.outputs.validation_response = validated_messages
-        if isinstance(validated_messages, ReAsk):
-            raise ValidationError(
-                f"Messages validation failed: " f"{validated_messages}"
+            validated_msg = validator_service.post_process_validation(
+                value, attempt_number, iteration, OutputTypes.STRING
             )
-        if validated_messages != msg_str:
-            raise ValidationError("Messages validation failed")
+            
+            iteration.outputs.validation_response = validated_msg
+
+            if isinstance(validated_msg, ReAsk):
+                raise ValidationError(f"Messages validation failed: {validated_msg}")
+            elif not validated_msg or iteration.status == fail_status:
+                raise ValidationError("Messages validation failed")
+
+            msg["content"] = cast(str, validated_msg)
+
+        return messages  # type: ignore

guardrails/run/runner.py

@@ -287,22 +287,17 @@ def step(
     def validate_messages(
         self, call_log: Call, messages: MessageHistory, attempt_number: int
     ) -> None:
-        msg_str = messages_string(messages)
-        inputs = Inputs(
-            llm_output=msg_str,
-        )
-        iteration = Iteration(call_id=call_log.id, index=attempt_number, inputs=inputs)
-        call_log.iterations.insert(0, iteration)
-
-        validated_msgs = ""
-
         for msg in messages:
             content = (
                 msg["content"].source
                 if isinstance(msg["content"], Prompt)
                 else msg["content"]
             )
-
+            inputs = Inputs(
+                        llm_output=content,
+                    )
+            iteration = Iteration(call_id=call_log.id, index=attempt_number, inputs=inputs)
+            call_log.iterations.insert(0, iteration)
             value, _metadata = validator_service.validate(
                 value=content,
                 metadata=self.metadata,
@@ -315,16 +310,15 @@ def validate_messages(
             validated_msg = validator_service.post_process_validation(
                 value, attempt_number, iteration, OutputTypes.STRING
             )
+            
+            iteration.outputs.validation_response = validated_msg
 
             if isinstance(validated_msg, ReAsk):
-                raise ValidationError(f"Message validation failed: {validated_msg}")
+                raise ValidationError(f"Messages validation failed: {validated_msg}")
             elif not validated_msg or iteration.status == fail_status:
-                raise ValidationError("Message validation failed")
+                raise ValidationError("Messages validation failed")
 
             msg["content"] = cast(str, validated_msg)
-            validated_msgs += validated_msg
-
-        iteration.outputs.validation_response = validated_msgs
 
         return messages  # type: ignore
 

tests/integration_tests/mock_llm_outputs.py

@@ -128,6 +128,11 @@ def _invoke_llm(
         }
 
         try:
+            if msg_history:
+                key = (msg_history[0]["content"], msg_history[1]["content"])
+                print("=========trying key", key)
+                out_text = mock_llm_responses[key]
+                print("========found out text", out_text)
             if prompt and instructions and not msg_history:
                 out_text = mock_llm_responses[(prompt, instructions)]
             elif msg_history and not prompt and not instructions:

tests/integration_tests/test_formatters.py

@@ -25,7 +25,7 @@ class Foo(BaseModel):
         bez: List[str]
 
     g = Guard.from_pydantic(Foo, output_formatter="jsonformer")
-    response = g(model.generate, tokenizer=tokenizer, prompt="test")
+    response = g(model.generate, tokenizer=tokenizer, messages=[{"content": "test","role": "user"}])
     validated_output = response.validated_output
     assert isinstance(validated_output, dict)
     assert "bar" in validated_output
@@ -45,7 +45,7 @@ class Foo(BaseModel):
         bez: List[str]
 
     g = Guard.from_pydantic(Foo, output_formatter="jsonformer")
-    response = g(model, prompt="Sample:")
+    response = g(model, messages=[{"content": "Sample:","role": "user"}])
     validated_output = response.validated_output
     assert isinstance(validated_output, dict)
     assert "bar" in validated_output

tests/integration_tests/test_parsing.py

@@ -49,16 +49,16 @@ def test_parsing_reask(mocker):
     assert call.iterations.length == 2
 
     # For orginal prompt and output
-    assert call.compiled_prompt == pydantic.PARSING_COMPILED_PROMPT
+    assert call.compiled_messages[0]["content"]._source == pydantic.PARSING_COMPILED_PROMPT
     assert call.iterations.first.raw_output == pydantic.PARSING_UNPARSEABLE_LLM_OUTPUT
     assert call.iterations.first.guarded_output is None
 
     # For re-asked prompt and output
-    assert call.iterations.last.inputs.prompt == gd.Prompt(
+    assert call.iterations.last.inputs.messages[1]["content"] == gd.Prompt(
         pydantic.PARSING_COMPILED_REASK
     )
     # Same as above
-    assert call.reask_prompts.last == pydantic.PARSING_COMPILED_REASK
+    assert call.reask_messages[0][1]["content"]._source == pydantic.PARSING_COMPILED_REASK
     assert call.raw_outputs.last == pydantic.PARSING_EXPECTED_LLM_OUTPUT
     assert call.guarded_output == pydantic.PARSING_EXPECTED_OUTPUT
 
@@ -83,7 +83,8 @@ async def test_async_parsing_reask(mocker):
     ]
 
     guard = gd.AsyncGuard.from_pydantic(
-        output_class=pydantic.PersonalDetails, prompt=pydantic.PARSING_INITIAL_PROMPT
+        output_class=pydantic.PersonalDetails,
+        messages=[{"role": "user", "content": pydantic.PARSING_INITIAL_PROMPT}],
     )
 
     final_output = await guard(
@@ -100,17 +101,17 @@ async def test_async_parsing_reask(mocker):
     assert call.iterations.length == 2
 
     # For orginal prompt and output
-    assert call.compiled_prompt == pydantic.PARSING_COMPILED_PROMPT
+    assert call.compiled_messages[0]["content"]._source == pydantic.PARSING_COMPILED_PROMPT
     assert call.iterations.first.raw_output == pydantic.PARSING_UNPARSEABLE_LLM_OUTPUT
     assert call.iterations.first.guarded_output is None
 
     # For re-asked prompt and output
 
-    assert call.iterations.last.inputs.prompt == gd.Prompt(
+    assert call.iterations.last.inputs.messages[1]["content"] == gd.Prompt(
         pydantic.PARSING_COMPILED_REASK
     )
     # Same as above
-    assert call.reask_prompts.last == pydantic.PARSING_COMPILED_REASK
+    assert call.reask_messages[0][1]["content"]._source == pydantic.PARSING_COMPILED_REASK
     assert call.raw_outputs.last == pydantic.PARSING_EXPECTED_LLM_OUTPUT
     assert call.guarded_output == pydantic.PARSING_EXPECTED_OUTPUT
 
@@ -123,7 +124,7 @@ def test_reask_prompt_instructions(mocker):
     """
 
     mocker.patch(
-        "guardrails.llm_providers.OpenAIChatCallable._invoke_llm",
+        "guardrails.llm_providers.LiteLLMCallable._invoke_llm",
         return_value=LLMResponse(
             output=string.MSG_LLM_OUTPUT_CORRECT,
             prompt_token_count=123,
@@ -144,7 +145,7 @@ def always_fail(value: str, metadata: Dict) -> ValidationResult:
 
     guard.parse(
         llm_output="Tomato Cheese Pizza",
-        llm_api=openai.chat.completions.create,
+        model="gpt-3.5-turbo",
         messages=[
             {"role": "system", "content": "Some content"},
             {"role": "user", "content": "Some prompt"},