Improve integration tests

Author: ericdallo

integration-test/integration/chat_openai_test.clj

@@ -27,16 +27,16 @@
       (llm.mocks/set-case! :simple-text-0)
       (let [req-id 0
             resp (eca/request! (fixture/chat-prompt-request
-                                 {:request-id req-id
-                                  :model "gpt-5"
-                                  :message "Tell me a joke!"}))
+                                {:request-id req-id
+                                 :model "gpt-5"
+                                 :message "Tell me a joke!"}))
             chat-id (reset! chat-id* (:chatId resp))]
 
         (is (match?
-              {:chatId (m/pred string?)
-               :model "gpt-5"
-               :status "success"}
-              resp))
+             {:chatId (m/pred string?)
+              :model "gpt-5"
+              :status "success"}
+             resp))
 
         (match-content chat-id req-id "user" {:type "text" :text "Tell me a joke!\n"})
         (match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
@@ -49,23 +49,27 @@
                                                 :sessionTokens 30
                                                 :messageCost (m/pred string?)
                                                 :sessionCost (m/pred string?)})
-        (match-content chat-id req-id "system" {:type "progress" :state "finished"})))
+        (match-content chat-id req-id "system" {:type "progress" :state "finished"})
+        (is (match?
+             {:input [{:role "user" :content [{:type "input_text" :text "Tell me a joke!"}]}]
+              :instructions (m/pred string?)}
+             llm.mocks/*last-req-body*))))
 
     (testing "We reply"
       (llm.mocks/set-case! :simple-text-1)
       (let [req-id 1
             resp (eca/request! (fixture/chat-prompt-request
-                                 {:chat-id @chat-id*
-                                  :request-id req-id
-                                  :model "gpt-5"
-                                  :message "Who's there?"}))
+                                {:chat-id @chat-id*
+                                 :request-id req-id
+                                 :model "gpt-5"
+                                 :message "Who's there?"}))
             chat-id @chat-id*]
 
         (is (match?
-              {:chatId (m/pred string?)
-               :model "gpt-5"
-               :status "success"}
-              resp))
+             {:chatId (m/pred string?)
+              :model "gpt-5"
+              :status "success"}
+             resp))
 
         (match-content chat-id req-id "user" {:type "text" :text "Who's there?\n"})
         (match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
@@ -77,23 +81,28 @@
                                                 :sessionTokens 45
                                                 :messageCost (m/pred string?)
                                                 :sessionCost (m/pred string?)})
-        (match-content chat-id req-id "system" {:type "progress" :state "finished"})))
+        (match-content chat-id req-id "system" {:type "progress" :state "finished"})
+        (is (match?
+             {:input [{:role "user" :content [{:type "input_text" :text "Tell me a joke!"}]}
+                      {:role "assistant" :content [{:type "output_text" :text "Knock knock!"}]}
+                      {:role "user" :content [{:type "input_text" :text "Who's there?"}]}]}
+             llm.mocks/*last-req-body*))))
 
     (testing "model reply again keeping context"
       (llm.mocks/set-case! :simple-text-2)
       (let [req-id 2
             resp (eca/request! (fixture/chat-prompt-request
-                                 {:chat-id @chat-id*
-                                  :request-id req-id
-                                  :model "gpt-5"
-                                  :message "What foo?"}))
+                                {:chat-id @chat-id*
+                                 :request-id req-id
+                                 :model "gpt-5"
+                                 :message "What foo?"}))
             chat-id @chat-id*]
 
         (is (match?
-              {:chatId (m/pred string?)
-               :model "gpt-5"
-               :status "success"}
-              resp))
+             {:chatId (m/pred string?)
+              :model "gpt-5"
+              :status "success"}
+             resp))
 
         (match-content chat-id req-id "user" {:type "text" :text "What foo?\n"})
         (match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
@@ -108,4 +117,11 @@
                                                 :sessionTokens 65
                                                 :messageCost (m/pred string?)
                                                 :sessionCost (m/pred string?)})
-        (match-content chat-id req-id "system" {:type "progress" :state "finished"})))))
+        (match-content chat-id req-id "system" {:type "progress" :state "finished"})
+        (is (match?
+             {:input [{:role "user" :content [{:type "input_text" :text "Tell me a joke!"}]}
+                      {:role "assistant" :content [{:type "output_text" :text "Knock knock!"}]}
+                      {:role "user" :content [{:type "input_text" :text "Who's there?"}]}
+                      {:role "assistant" :content [{:type "output_text" :text "Foo"}]}
+                      {:role "user" :content [{:type "input_text" :text "What foo?"}]}]}
+             llm.mocks/*last-req-body*))))))

integration-test/integration/eca.clj

@@ -4,7 +4,8 @@
    [clojure.core.async :as async]
    [clojure.java.io :as io]
    [clojure.test :refer [use-fixtures]]
-   [integration.client :as client]))
+   [integration.client :as client]
+   [llm-mock.mocks :as llm.mocks]))
 
 (def ^:dynamic *eca-binary-path* nil)
 (def ^:dynamic *eca-process* nil)
@@ -33,7 +34,9 @@
   (some-> *mock-client* client/shutdown)
   (some-> *eca-process* deref) ;; wait for shutdown of client to shutdown server
   (alter-var-root #'*eca-process* (constantly nil))
-  (alter-var-root #'*mock-client* (constantly nil)))
+  (alter-var-root #'*mock-client* (constantly nil))
+  (llm.mocks/set-case! nil)
+  (llm.mocks/set-last-req-body! nil))
 
 (defn clean-after-test []
   (use-fixtures :each (fn [f] (clean!) (f)))

integration-test/integration/fixture.clj

@@ -9,7 +9,9 @@
 
 (def default-init-options {:pureConfig true
                            :openaiApiUrl (str base-llm-mock-url "/openai")
-                           :anthropicApiUrl (str base-llm-mock-url "/anthropic")})
+                           :openaiApiKey "foo"
+                           :anthropicApiUrl (str base-llm-mock-url "/anthropic")
+                           :anthropicApiKey "foo"})
 
 (defn initialize-request
   ([]

integration-test/llm_mock/mocks.clj

@@ -4,3 +4,8 @@
 
 (defn set-case! [case]
   (alter-var-root #'*case* (constantly case)))
+
+(def ^:dynamic *last-req-body* nil)
+
+(defn set-last-req-body! [body]
+  (alter-var-root #'*last-req-body* (constantly body)))

integration-test/llm_mock/openai.clj

@@ -53,6 +53,8 @@
   (hk/close ch))
 
 (defn handle-openai-responses [req]
+  (llm.mocks/set-last-req-body! (some-> (slurp (:body req))
+                                        (json/parse-string true)))
   (hk/as-channel
    req
    {:on-open (fn [ch]