safeguard static openai funciton checks; update async tests to use custom llm

Author: CalebCourier

guardrails/applications/text2sql.py

@@ -1,14 +1,14 @@
 import asyncio
 import json
 import os
+import openai
 from string import Template
 from typing import Callable, Dict, Optional, Type, cast
 
 from guardrails.classes import ValidationOutcome
 from guardrails.document_store import DocumentStoreBase, EphemeralDocumentStore
 from guardrails.embedding import EmbeddingBase, OpenAIEmbedding
 from guardrails.guard import Guard
-from guardrails.utils.openai_utils import get_static_openai_create_func
 from guardrails.utils.sql_utils import create_sql_driver
 from guardrails.vectordb import Faiss, VectorDBBase
 
@@ -89,7 +89,7 @@ def __init__(
             reask_prompt: Prompt to use for reasking. Defaults to REASK_PROMPT.
         """
         if llm_api is None:
-            llm_api = get_static_openai_create_func()
+            llm_api = openai.completions.create
 
         self.example_formatter = example_formatter
         self.llm_api = llm_api

guardrails/llm_providers.py

@@ -27,10 +27,10 @@
 from guardrails.utils.openai_utils import (
     AsyncOpenAIClient,
     OpenAIClient,
-    get_static_openai_acreate_func,
-    get_static_openai_chat_acreate_func,
-    get_static_openai_chat_create_func,
-    get_static_openai_create_func,
+    is_static_openai_acreate_func,
+    is_static_openai_chat_acreate_func,
+    is_static_openai_chat_create_func,
+    is_static_openai_create_func,
 )
 from guardrails.utils.pydantic_utils import convert_pydantic_model_to_openai_fn
 from guardrails.utils.safe_get import safe_get
@@ -784,9 +784,9 @@ def get_llm_ask(
     except ImportError:
         pass
 
-    if llm_api == get_static_openai_create_func():
+    if is_static_openai_create_func(llm_api):
         return OpenAICallable(*args, **kwargs)
-    if llm_api == get_static_openai_chat_create_func():
+    if is_static_openai_chat_create_func(llm_api):
         return OpenAIChatCallable(*args, **kwargs)
 
     try:
@@ -1252,9 +1252,12 @@ def get_async_llm_ask(
         pass
 
     # these only work with openai v0 (None otherwise)
-    if llm_api == get_static_openai_acreate_func():
+    # We no longer support OpenAI v0
+    # We should drop these checks or update the logic to support
+    #   OpenAI v1 clients instead of just static methods
+    if is_static_openai_acreate_func(llm_api):
         return AsyncOpenAICallable(*args, **kwargs)
-    if llm_api == get_static_openai_chat_acreate_func():
+    if is_static_openai_chat_acreate_func(llm_api):
         return AsyncOpenAIChatCallable(*args, **kwargs)
 
     try:
@@ -1293,13 +1296,13 @@ def get_llm_api_enum(
 ) -> Optional[LLMResource]:
     # TODO: Distinguish between v1 and v2
     model = get_llm_ask(llm_api, *args, **kwargs)
-    if llm_api == get_static_openai_create_func():
+    if is_static_openai_create_func(llm_api):
         return LLMResource.OPENAI_DOT_COMPLETION_DOT_CREATE
-    elif llm_api == get_static_openai_chat_create_func():
+    elif is_static_openai_chat_create_func(llm_api):
         return LLMResource.OPENAI_DOT_CHAT_COMPLETION_DOT_CREATE
-    elif llm_api == get_static_openai_acreate_func():
+    elif is_static_openai_acreate_func(llm_api):  # This is always False
         return LLMResource.OPENAI_DOT_COMPLETION_DOT_ACREATE
-    elif llm_api == get_static_openai_chat_acreate_func():
+    elif is_static_openai_chat_acreate_func(llm_api):  # This is always False
         return LLMResource.OPENAI_DOT_CHAT_COMPLETION_DOT_ACREATE
     elif isinstance(model, LiteLLMCallable):
         return LLMResource.LITELLM_DOT_COMPLETION

guardrails/utils/openai_utils/__init__.py

@@ -2,18 +2,18 @@
 from .v1 import OpenAIClientV1 as OpenAIClient
 from .v1 import (
     OpenAIServiceUnavailableError,
-    get_static_openai_acreate_func,
-    get_static_openai_chat_acreate_func,
-    get_static_openai_chat_create_func,
-    get_static_openai_create_func,
+    is_static_openai_acreate_func,
+    is_static_openai_chat_acreate_func,
+    is_static_openai_chat_create_func,
+    is_static_openai_create_func,
 )
 
 __all__ = [
     "AsyncOpenAIClient",
     "OpenAIClient",
-    "get_static_openai_create_func",
-    "get_static_openai_chat_create_func",
-    "get_static_openai_acreate_func",
-    "get_static_openai_chat_acreate_func",
+    "is_static_openai_create_func",
+    "is_static_openai_chat_create_func",
+    "is_static_openai_acreate_func",
+    "is_static_openai_chat_acreate_func",
     "OpenAIServiceUnavailableError",
 ]

guardrails/utils/openai_utils/v1.py

@@ -1,4 +1,4 @@
-from typing import Any, AsyncIterable, Dict, Iterable, List, cast
+from typing import Any, AsyncIterable, Callable, Dict, Iterable, List, Optional, cast
 
 import openai
 
@@ -12,20 +12,30 @@
 from guardrails.telemetry import trace_llm_call, trace_operation
 
 
-def get_static_openai_create_func():
-    return openai.completions.create
+def is_static_openai_create_func(llm_api: Optional[Callable]) -> bool:
+    try:
+        return llm_api == openai.completions.create
+    except openai.OpenAIError:
+        return False
 
 
-def get_static_openai_chat_create_func():
-    return openai.chat.completions.create
+def is_static_openai_chat_create_func(llm_api: Optional[Callable]) -> bool:
+    try:
+        return llm_api == openai.chat.completions.create
+    except openai.OpenAIError:
+        return False
 
 
-def get_static_openai_acreate_func():
-    return None
+def is_static_openai_acreate_func(llm_api: Optional[Callable]) -> bool:
+    # Because the static version of this does not exist in OpenAI 1.x
+    # Can we just drop these checks?
+    return False
 
 
-def get_static_openai_chat_acreate_func():
-    return None
+def is_static_openai_chat_acreate_func(llm_api: Optional[Callable]) -> bool:
+    # Because the static version of this does not exist in OpenAI 1.x
+    # Can we just drop these checks?
+    return False
 
 
 OpenAIServiceUnavailableError = openai.APIError

tests/integration_tests/test_assets/entity_extraction/__init__.py

@@ -33,10 +33,12 @@
 
 # Compiled prompts
 COMPILED_PROMPT = reader("compiled_prompt.txt")
+NON_OPENAI_COMPILED_PROMPT = reader("non_openai_compiled_prompt.txt")
 COMPILED_PROMPT_WITHOUT_INSTRUCTIONS = reader(
     "compiled_prompt_without_instructions.txt"
 )
 COMPILED_PROMPT_REASK = reader("compiled_prompt_reask.txt")
+NON_OPENAI_COMPILED_PROMPT_REASK = reader("non_openai_compiled_prompt_reask.txt")
 COMPILED_PROMPT_REASK_WITHOUT_INSTRUCTIONS = reader(
     "compiled_prompt_reask_without_instructions.txt"
 )
@@ -82,8 +84,10 @@
 
 __all__ = [
     "COMPILED_PROMPT",
+    "NON_OPENAI_COMPILED_PROMPT",
     "COMPILED_PROMPT_WITHOUT_INSTRUCTIONS",
     "COMPILED_PROMPT_REASK",
+    "NON_OPENAI_COMPILED_PROMPT_REASK",
     "COMPILED_PROMPT_REASK_WITHOUT_INSTRUCTIONS",
     "COMPILED_INSTRUCTIONS",
     "COMPILED_INSTRUCTIONS_REASK",

tests/integration_tests/test_assets/entity_extraction/non_openai_compiled_prompt.txt

@@ -0,0 +1,131 @@
+
+Given the following document, answer the following questions. If the answer doesn't exist in the document, enter 'None'.
+
+2/25/23, 7:59 PM about:blank
+about:blank 1/4
+PRICING INFORMATION
+INTEREST RATES AND INTEREST CHARGES
+Purchase Annual
+Percentage Rate (APR) 0% Intro APR for the first 18 months that your Account is open.
+After that, 19.49%. This APR will vary with the market based on the Prime
+Rate.
+a
+My Chase Loan
+SM APR 19.49%. This APR will vary with the market based on the Prime Rate.
+a
+Promotional offers with fixed APRs and varying durations may be available from
+time to time on some accounts.
+Balance Transfer APR 0% Intro APR for the first 18 months that your Account is open.
+After that, 19.49%. This APR will vary with the market based on the Prime
+Rate.
+a
+Cash Advance APR 29.49%. This APR will vary with the market based on the Prime Rate.
+b
+Penalty APR and When
+It Applies
+Up to 29.99%. This APR will vary with the market based on the Prime Rate.
+c
+We may apply the Penalty APR to your account if you:
+fail to make a Minimum Payment by the date and time that it is due; or
+make a payment to us that is returned unpaid.
+How Long Will the Penalty APR Apply?: If we apply the Penalty APR for
+either of these reasons, the Penalty APR could potentially remain in effect
+indefinitely.
+How to Avoid Paying
+Interest on Purchases
+Your due date will be a minimum of 21 days after the close of each billing cycle.
+We will not charge you interest on new purchases if you pay your entire balance
+or Interest Saving Balance by the due date each month. We will begin charging
+interest on balance transfers and cash advances on the transaction date.
+Minimum Interest
+Charge
+None
+Credit Card Tips from
+the Consumer Financial
+Protection Bureau
+To learn more about factors to consider when applying for or using a credit card,
+visit the website of the Consumer Financial Protection Bureau at
+http://www.consumerfinance.gov/learnmore.
+FEES
+Annual Membership
+Fee
+None
+My Chase Plan
+SM Fee
+(fixed finance charge)
+Monthly fee of 0% of the amount of each eligible purchase transaction or
+amount selected to create a My Chase Plan while in the 0% Intro Purchase
+APR period.
+After that, monthly fee of 1.72% of the amount of each eligible purchase
+transaction or amount selected to create a My Chase Plan. The My Chase Plan
+Fee will be determined at the time each My Chase Plan is created and will
+remain the same until the My Chase Plan is paid in full.
+d
+Transaction Fees
+Balance Transfers Intro fee of either $5 or 3% of the amount of each transfer, whichever is greater,
+on transfers made within 60 days of account opening. After that: Either $5 or 5%
+of the amount of each transfer, whichever is greater.
+Cash Advances Either $10 or 5% of the amount of each transaction, whichever is greater.
+2/25/23, 7:59 PM about:blank
+about:blank 2/4
+Foreign Transactions 3% of the amount of each transaction in U.S. dollars.
+Penalty Fees
+Late Payment Up to $40.
+Over-the-Credit-Limit None
+Return Payment Up to $40.
+Return Check None
+Note: This account may not be eligible for balance transfers.
+Loss of Intro APR: We will end your introductory APR if any required Minimum Payment is 60 days late, and
+apply the Penalty APR.
+How We Will Calculate Your Balance: We use the daily balance method (including new transactions).
+Prime Rate: Variable APRs are based on the 7.75% Prime Rate as of 2/7/2023.
+aWe add 11.74% to the Prime Rate to determine the Purchase/My Chase Loan/Balance Transfer APR.
+Maximum APR 29.99%.
+bWe add 21.74% to the Prime Rate to determine the Cash Advance APR. Maximum APR 29.99%.
+cWe add up to 26.99% to the Prime Rate to determine the Penalty APR. Maximum APR 29.99%.
+dMy Chase Plan Fee: The My Chase Plan Fee is calculated at the time each plan is created and is based on
+the amount of each purchase transaction or amount selected to create the plan, the number of billing periods
+you choose to pay the balance in full, and other factors. The monthly and aggregate dollar amount of your My
+Chase Plan Fee will be disclosed during the activation of each My Chase Plan.
+MILITARY LENDING ACT NOTICE: Federal law provides important protections to members of the Armed
+Forces and their dependents relating to extensions of consumer credit. In general, the cost of consumer credit
+to a member of the Armed Forces and his or her dependent may not exceed an annual percentage rate of 36
+percent. This rate must include, as applicable to the credit transaction or account: the costs associated with
+credit insurance premiums; fees for ancillary products sold in connection with the credit transaction; any
+application fee charged (other than certain application fees for specified credit transactions or accounts); and
+any participation fee charged (other than certain participation fees for a credit card account). To receive this
+information and a description of your payment obligation verbally, please call 1-800-235-9978.
+TERMS & CONDITIONS
+Authorization: When you respond to this credit card offer from JPMorgan Chase Bank, N.A., Member FDIC, a
+subsidiary of JPMorgan Chase & Co. ("Chase", "we", or "us"), you agree to the following:
+1. You authorize us to obtain credit bureau reports, employment, and income information about you that we
+will use when considering your application for credit. We may obtain and use information about your
+accounts with us and others such as Checking, Deposit, Investment, and Utility accounts from credit
+bureaus and other entities. You also authorize us to obtain credit bureau reports and any other
+information about you in connection with: 1) extensions of credit on your account; 2) the administration,
+review or collection of your account; and 3) offering you enhanced or additional products and services. If
+you ask, we will tell you the name and address of the credit bureau from which we obtained a report
+about you.
+2. If an account is opened, you will receive a Cardmember Agreement with your card(s). You agree to the
+terms of this agreement by: using the account or any card, authorizing their use, or making any payment
+on the account.
+3. By providing your mobile ph
+
+
+Given below is XML that describes the information to extract from this document and the tags to extract it into.
+
+
+<output>
+  <list description="What fees and charges are associated with my account?" name="fees" required="true">
+    <object required="true">
+      <integer format="1-indexed" name="index" required="true"></integer>
+      <string format="lower-case; two-words" name="name" required="true"></string>
+      <string format="one-line" name="explanation" required="true"></string>
+      <float format="percentage" name="value" required="true"></float>
+    </object>
+  </list>
+  <object description="What are the interest rates offered by the bank on savings and checking accounts, loans, and credit products?" name="interest_rates" required="true"></object>
+</output>
+
+
+ONLY return a valid JSON object (no other text is necessary). The JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be correct and concise.

tests/integration_tests/test_assets/entity_extraction/non_openai_compiled_prompt_reask.txt

@@ -0,0 +1,41 @@
+
+I was given the following JSON response, which had problems due to incorrect values.
+
+{
+  "fees": [
+    {
+      "name": {
+        "incorrect_value": "my chase plan",
+        "error_messages": [
+          "must be exactly two words"
+        ]
+      }
+    },
+    {
+      "name": {
+        "incorrect_value": "over-the-credit-limit",
+        "error_messages": [
+          "must be exactly two words"
+        ]
+      }
+    }
+  ]
+}
+
+Help me correct the incorrect values based on the given error messages.
+
+Given below is XML that describes the information to extract from this document and the tags to extract it into.
+
+<output>
+  <list description="What fees and charges are associated with my account?" name="fees" required="true">
+    <object required="true">
+      <integer format="1-indexed" name="index" required="true"></integer>
+      <string format="lower-case; two-words" name="name" required="true"></string>
+      <string format="one-line" name="explanation" required="true"></string>
+      <float format="percentage" name="value" required="true"></float>
+    </object>
+  </list>
+  <object description="What are the interest rates offered by the bank on savings and checking accounts, loans, and credit products?" name="interest_rates" required="true"></object>
+</output>
+
+ONLY return a valid JSON object (no other text is necessary), where the key of the field in JSON is the `name` attribute of the corresponding XML, and the value is of the type specified by the corresponding XML's tag. The JSON MUST conform to the XML format, including any types and format requests e.g. requests for lists, objects and specific types. Be correct and concise. If you are unsure anywhere, enter `null`.