tests passing

Author: dtam

guardrails/classes/history/call.py

@@ -16,6 +16,7 @@
 from guardrails.classes.validation.validation_result import ValidationResult
 from guardrails.constants import error_status, fail_status, not_run_status, pass_status
 from guardrails.prompt.messages import Messages
+from guardrails.prompt import Prompt, Instructions
 from guardrails.classes.validation.validator_logs import ValidatorLogs
 from guardrails.actions.reask import (
     ReAsk,
@@ -93,10 +94,13 @@ def compiled_messages(self) -> Optional[str]:
         prompt_params = initial_inputs.prompt_params or {}
         compiled_messages = []
         for message in messages:
+            content = message["content"].format(**prompt_params)
+            if isinstance(content, (Prompt, Instructions)):
+                content = content._source
             compiled_messages.append(
                 {
                     "role": message["role"],
-                    "content": message["content"].format(**prompt_params),
+                    "content": content,
                 }
             )
 
@@ -112,12 +116,28 @@ def reask_messages(self) -> Stack[Messages]:
             reasks = self.iterations.copy()
             initial_messages = reasks.first
             reasks.remove(initial_messages)  # type: ignore
-            return Stack(
-                *[
-                    r.inputs.messages if r.inputs.messages is not None else None
-                    for r in reasks
-                ]
-            )
+            initial_inputs = self.iterations.first.inputs
+            prompt_params = initial_inputs.prompt_params or {}
+            compiled_reasks = []
+            for reask in reasks:
+                messages: Messages = reask.inputs.messages
+
+                if messages is None:
+                    compiled_reasks.append(None)
+                else:
+                    compiled_messages = []
+                    for message in messages:
+                        content = message["content"].format(**prompt_params)
+                        if isinstance(content, (Prompt, Instructions)):
+                            content = content._source
+                        compiled_messages.append(
+                            {
+                                "role": message["role"],
+                                "content": content,
+                            }
+                        )
+                compiled_reasks.append(compiled_messages)
+            return Stack(*compiled_reasks)
 
         return Stack()
 

guardrails/classes/history/inputs.py

@@ -9,6 +9,7 @@
 from guardrails.prompt.messages import Messages
 from guardrails.prompt.instructions import Instructions
 
+
 class Inputs(IInputs, ArbitraryModel):
     """Inputs represent the input data that is passed into the validation loop.
 
@@ -38,7 +39,9 @@ class Inputs(IInputs, ArbitraryModel):
         "provided by the user via Guard.parse.",
         default=None,
     )
-    messages: Optional[Union[List[Dict[str, Union[str, Prompt, Instructions]]], Messages]] = Field(
+    messages: Optional[
+        Union[List[Dict[str, Union[str, Prompt, Instructions]]], Messages]
+    ] = Field(
         description="The message history provided by the user for chat model calls.",
         default=None,
     )
@@ -95,7 +98,6 @@ def to_dict(self) -> Dict[str, Any]:
     @classmethod
     def from_interface(cls, i_inputs: IInputs) -> "Inputs":
         deserialized_messages = None
-        print("====== inputs.py: Inputs.from_interface() ======", i_inputs)
         if i_inputs.messages:
             deserialized_messages = []
             for msg in i_inputs.messages:

guardrails/guard.py

@@ -841,7 +841,7 @@ def __call__(
         """
 
         messages = messages or self._exec_opts.messages or []
-        print("==== messages is", messages)
+
         # if messages is not None and not len(messages):
         #     raise RuntimeError(
         #         "You must provide messages. "

guardrails/llm_providers.py

@@ -13,7 +13,7 @@
     cast,
 )
 
-from guardrails.prompt import Prompt
+from guardrails.prompt import Prompt, Instructions
 from guardrails_api_client.models import LLMResource
 
 from guardrails.errors import UserFacingException
@@ -25,6 +25,12 @@
 
 from guardrails.types.inputs import MessageHistory
 
+import warnings
+
+from guardrails.utils.safe_get import safe_get
+from guardrails.telemetry import trace_llm_call, trace_operation
+
+
 # todo fix circular import
 def messages_string(messages: MessageHistory) -> str:
     messages_copy = ""
@@ -37,9 +43,6 @@ def messages_string(messages: MessageHistory) -> str:
         messages_copy += content
     return messages_copy
 
-from guardrails.utils.safe_get import safe_get
-from guardrails.telemetry import trace_llm_call, trace_operation
-
 
 ###
 # Synchronous wrappers
@@ -183,9 +186,7 @@ def _invoke_llm(
                 "Install with `pip install litellm`"
             ) from e
         if messages is not None:
-            messages = litellm_messages(
-                prompt=text, messages=messages
-            )
+            messages = litellm_messages(prompt=text, messages=messages)
             kwargs["messages"] = messages
 
         trace_operation(
@@ -265,7 +266,11 @@ def _invoke_llm(
 
 class HuggingFaceModelCallable(PromptCallableBase):
     def _invoke_llm(
-        self, model_generate: Any, *args, messages: list[dict[str, str]], **kwargs
+        self,
+        model_generate: Any,
+        *args,
+        messages: list[dict[str, Union[str, Prompt, Instructions]]],
+        **kwargs,
     ) -> LLMResponse:
         try:
             import transformers  # noqa: F401 # type: ignore
@@ -431,6 +436,14 @@ def __init__(self, llm_api: Optional[Callable] = None, *args, **kwargs):
         llm_api_args = inspect.getfullargspec(llm_api)
         if not llm_api_args.varkw:
             raise ValueError("Custom LLM callables must accept **kwargs!")
+        if not llm_api_args.kwonlyargs or "messages" not in llm_api_args.kwonlyargs:
+            warnings.warn(
+                "We recommend including 'messages'"
+                " as keyword-only arguments for custom LLM callables."
+                " Doing so ensures these arguments are not unintentionally"
+                " passed through to other calls via **kwargs.",
+                UserWarning,
+            )
         self.llm_api = llm_api
         super().__init__(*args, **kwargs)
 
@@ -771,13 +784,16 @@ async def invoke_llm(
 class AsyncArbitraryCallable(AsyncPromptCallableBase):
     def __init__(self, llm_api: Callable, *args, **kwargs):
         llm_api_args = inspect.getfullargspec(llm_api)
-        if not llm_api_args.args:
-            raise ValueError(
-                "Custom LLM callables must accept"
-                " at least one positional argument for messages!"
-            )
         if not llm_api_args.varkw:
             raise ValueError("Custom LLM callables must accept **kwargs!")
+        if not llm_api_args.kwonlyargs or "messages" not in llm_api_args.kwonlyargs:
+            warnings.warn(
+                "We recommend including 'messages'"
+                " as keyword-only arguments for custom LLM callables."
+                " Doing so ensures these arguments are not unintentionally"
+                " passed through to other calls via **kwargs.",
+                UserWarning,
+            )
         self.llm_api = llm_api
         super().__init__(*args, **kwargs)
 

tests/integration_tests/mock_llm_outputs.py

@@ -128,26 +128,15 @@ def _invoke_llm(
         }
 
         try:
+            out_text = None
             if messages:
                 if len(messages) == 2:
                     key = (messages[0]["content"], messages[1]["content"])
                 elif len(messages) == 1:
                     key = (messages[0]["content"], None)
-                out_text = mock_llm_responses[key]
-            if prompt and instructions and not messages:
-                out_text = mock_llm_responses[(prompt, instructions)]
-            elif messages and not prompt and not instructions:
-                if messages == entity_extraction.COMPILED_MSG_HISTORY:
-                    out_text = entity_extraction.LLM_OUTPUT
-                elif (
-                    messages == string.MOVIE_MSG_HISTORY
-                    and base_model == pydantic.WITH_MSG_HISTORY
-                ):
-                    out_text = pydantic.MSG_HISTORY_LLM_OUTPUT_INCORRECT
-                elif messages == string.MOVIE_MSG_HISTORY:
-                    out_text = string.MSG_LLM_OUTPUT_INCORRECT
-                else:
-                    raise ValueError("messages not found")
+
+                if hasattr(mock_llm_responses[key], "read"):
+                    out_text = mock_llm_responses[key]
             else:
                 raise ValueError(
                     "specify either prompt and instructions " "or messages"
@@ -162,7 +151,8 @@ def _invoke_llm(
             print("\n prompt: \n", prompt)
             print("\n instructions: \n", instructions)
             print("\n messages: \n", messages)
-            raise ValueError("Compiled prompt not found")
+            print("\n base_model: \n", base_model)
+            raise ValueError("Compiled prompt not found in mock llm response")
 
 
 class MockArbitraryCallable(ArbitraryCallable):

tests/integration_tests/test_assets/entity_extraction/compiled_prompt.txt

@@ -129,7 +129,3 @@ Given below is XML that describes the information to extract from this document
 
 
 ONLY return a valid JSON object (no other text is necessary). The JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be correct and concise.
-
-
-Json Output:
-

tests/integration_tests/test_assets/entity_extraction/compiled_prompt_full_reask.txt

@@ -110,7 +110,3 @@ Given below is XML that describes the information to extract from this document
 </output>
 
 ONLY return a valid JSON object (no other text is necessary), where the key of the field in JSON is the `name` attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, enter `null`.
-
-
-Json Output:
-

tests/integration_tests/test_assets/entity_extraction/compiled_prompt_reask.txt

@@ -39,7 +39,3 @@ Given below is XML that describes the information to extract from this document
 </output>
 
 ONLY return a valid JSON object (no other text is necessary), where the key of the field in JSON is the `name` attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, enter `null`.
-
-
-Json Output:
-

tests/integration_tests/test_assets/entity_extraction/compiled_prompt_skeleton_reask_1.txt

@@ -129,7 +129,3 @@ Given below is XML that describes the information to extract from this document
 
 
 ONLY return a valid JSON object (no other text is necessary). The JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be correct and concise.
-
-
-Json Output:
-

tests/integration_tests/test_assets/entity_extraction/compiled_prompt_skeleton_reask_2.txt

@@ -97,7 +97,3 @@ Here's an example of the structure:
   ],
   "interest_rates": {}
 }
-
-
-Json Output:
-