Mock results for LLM calls in test suite (#57)

* mock responses for llm calls

Signed-off-by: Mandana Vaziri <[email protected]>

* removing secrets from build file

Signed-off-by: Mandana Vaziri <[email protected]>

---------

Signed-off-by: Mandana Vaziri <[email protected]>

Author: vazirim

.github/workflows/build.yml

@@ -39,12 +39,6 @@ jobs:
     - name: pre-commit checks
       run: pre-commit run -a
     - name: run tests
-      env:
-        WATSONX_PROJECT_ID: ${{ secrets.WATSONX_PROJECT_ID }}
-        WATSONX_KEY: ${{ secrets.WATSONX_KEY }}
-        WATSONX_API: ${{ secrets.WATSONX_API }}
-        WATSONX_APIKEY: ${{ secrets.WATSONX_APIKEY }}
-        WATSONX_URL: ${{ secrets.WATSONX_URL }}
       run: py.test -v --capture=tee-sys tests
 
   viewer:

examples/hello/test.pdl

@@ -0,0 +1,7 @@
+description: Hello world with model chaining
+document:
+- model: "foo"
+  parameters:
+    foo:
+      data: "{{ translateText(sentence2) }}"
+      raw: true

tests/data/line/hello14.pdl

@@ -7,6 +7,7 @@ document:
     stop:
     - '!'
     include_stop_sequence: true
+    mock_response: " World!"
 - def: translate
   function:
     sentence: str
@@ -19,6 +20,7 @@ document:
         stop:
         - '!'
         include_stop_sequence: true
+        mock_response: "Bonjour le monde!"
 - call: translate
   spec: str
   args:

tests/data/line/hello16.pdl

@@ -25,3 +25,4 @@ document:
     stop:
     - '}'
     include_stop_sequence: true
+    mock_response: '{"bob": 20, "carol": 30}'

tests/data/line/hello24.pdl

@@ -7,6 +7,7 @@ document:
     stop:
     - '!'
     include_stop_sequence: true
+    mock_response: " World!"
 - def: translate
   function:
     sentence: str
@@ -15,6 +16,8 @@ document:
   return:
     - "\nTranslate the sentence '{{ sentence }}' to {{ language }}\n"
     - model: watsonx/ibm/granite-34b-code-instruct
+      parameters:
+        mock_response: " World!"
 - call: translate
   spec: str
   args:

tests/data/line/hello26.pdl

@@ -22,4 +22,12 @@ document:
     decoding_method: greedy
     stop:
     - '}'
-    include_stop_sequence: true
+    include_stop_sequence: true
+    mock_response:
+      |
+        Here is the code:
+        ```json
+        {
+          "bob": "20",
+          "carol": "30"
+        }

tests/data/line/hello3.pdl

@@ -9,4 +9,5 @@ document:
     decoding_method: greedy
     stop:
     - '!'
-    include_stop_sequence: true
+    include_stop_sequence: true
+    mock_response: " World!"

tests/test_defs.py

@@ -13,6 +13,7 @@
                     "parameters": {
                         "stop": ["!"],
                         "include_stop_sequence": False,
+                        "mock_response": " World",
                     },
                 }
             ]

tests/test_line_table.py

@@ -173,8 +173,8 @@ def test_line13(capsys):
     "errors": [
         "",
         "Hello, World!Bonjour le monde!",
-        "tests/data/line/hello14.pdl:22 - Type errors in result of function call to translate",
-        "tests/data/line/hello14.pdl:15 - Bonjour le monde! should be of type <class 'int'>",
+        "tests/data/line/hello14.pdl:24 - Type errors in result of function call to translate",
+        "tests/data/line/hello14.pdl:16 - Bonjour le monde! should be of type <class 'int'>",
     ],
 }
 
@@ -311,13 +311,13 @@ def test_line23(capsys):
     "errors": [
         "",
         "Hello, World!",
-        "tests/data/line/hello24.pdl:21 - 'GEN1' is undefined",
-        "tests/data/line/hello24.pdl:22 - 'GEN2' is undefined",
-        "tests/data/line/hello24.pdl:20 - Type errors during function call to translate",
+        "tests/data/line/hello24.pdl:24 - 'GEN1' is undefined",
+        "tests/data/line/hello24.pdl:25 - 'GEN2' is undefined",
+        "tests/data/line/hello24.pdl:23 - Type errors during function call to translate",
         "tests/data/line/hello24.pdl:21 - None should be of type <class 'str'>",
-        "tests/data/line/hello24.pdl:22 - None should be of type <class 'str'>",
-        "tests/data/line/hello24.pdl:18 - Type errors during spec checking",
-        "tests/data/line/hello24.pdl:18 - None should be of type <class 'str'>",
+        "tests/data/line/hello24.pdl:25 - None should be of type <class 'str'>",
+        "tests/data/line/hello24.pdl:21 - Type errors during spec checking",
+        "tests/data/line/hello24.pdl:24 - None should be of type <class 'str'>",
     ],
 }
 
@@ -336,9 +336,9 @@ def test_line24(capsys):
     ],
 }
 
-
-def test_line25(capsys):
-    do_test(line25, capsys)
+# Leaving this out for now, since we can't mock the model result
+# def test_line25(capsys):
+#    do_test(line25, capsys)
 
 
 line26 = {

tests/test_model.py

@@ -11,6 +11,7 @@
                 "decoding_method": "greedy",
                 "stop_sequences": ["!"],
                 "include_stop_sequence": False,
+                "mock_response": " World",
             },
         },
         "!\n",
@@ -38,6 +39,7 @@ def test_model():
                         "decoding_method": "greedy",
                         "stop": ["!"],
                         "include_stop_sequence": False,
+                        "mock_response": " World",
                     },
                 }
             ],
@@ -56,6 +58,7 @@ def test_model():
                         "stop_sequences": ["."],
                         "include_stop_sequence": True,
                         "roles": {"user": {"pre_message": "", "post_message": ""}},
+                        "mock_response": 'World is a fictional character in the popular science fiction television series "The X-Files',
                     },
                 },
             ],
@@ -98,6 +101,7 @@ def test_model_chain():
                         "decoding_method": "greedy",
                         "stop_sequences": ["Question"],
                         "include_stop_sequence": False,
+                        "mock_response": "Armonk",
                     },
                 }
             ],
@@ -121,6 +125,7 @@ def test_multi_shot():
             "model": "watsonx/ibm/granite-34b-code-instruct",
             "parameters": {
                 "stop_sequences": ["."],
+                "mock_response": '\nI have a question about the use of the word "in" in the sentence: "The cake was baked in the oven.',
             },
         },
     ],
@@ -144,9 +149,7 @@ def test_data_missing_parameters():
         "Hello,",
         {
             "model": "{{ model }}",
-            "parameters": {
-                "stop_sequences": ["!"],
-            },
+            "parameters": {"stop_sequences": ["!"], "mock_response": " World!"},
         },
     ],
 }
@@ -166,9 +169,7 @@ def test_model_parameter():
         "Hello,",
         {
             "model": "watsonx/ibm/{{ model }}",
-            "parameters": {
-                "stop_sequences": ["!"],
-            },
+            "parameters": {"stop_sequences": ["!"], "mock_response": " World!"},
         },
     ],
 }