Add first real chat integration-tests 🎉

Author: ericdallo

docs/configuration.md

@@ -196,6 +196,8 @@ Example:
 interface Config {
     openaiApiKey?: string;
     anthropicApiKey?: string;
+    openaiApiUrl?: string;
+    anthropicApiUrl?: string;
     rules: [{path: string;}];
     commands: [{path: string;}];
     systemPromptTemplateFile?: string;
@@ -250,6 +252,8 @@ interface Config {
 {
   "openaiApiKey" : null,
   "anthropicApiKey" : null,
+  "openaiApiUrl" : null,
+  "anthropicApiUrl" : null,
   "rules" : [],
   "commands" : [],
   "nativeTools": {"filesystem": {"enabled": true},
@@ -259,8 +263,8 @@ interface Config {
   "toolCall": {
     "manualApproval": null,
   },
-  "mcpTimeoutSeconds" : 10,
-  "mcpServers" : [],
+  "mcpTimeoutSeconds" : 60,
+  "mcpServers" : {},
   "customProviders": {},
   "models": {},
   "ollama" : {
@@ -270,7 +274,7 @@ interface Config {
     "think": true
   },
   "chat" : {
-    "welcomeMessage" : "Welcome to ECA! What you have in mind?\n\n"
+    "welcomeMessage" : "Welcome to ECA!\n\nType '/' for commands\n\n"
   },
   "index" : {
     "ignoreFiles" : [ {

integration-test/entrypoint.clj

@@ -1,7 +1,8 @@
 (ns entrypoint
   (:require
    [clojure.test :as t]
-   [integration.eca :as eca]))
+   [integration.eca :as eca]
+   [llm-mock.server :as llm-mock.server]))
 
 (def namespaces
   '[integration.initialize-test
@@ -33,13 +34,17 @@
   (alter-var-root #'eca/*eca-binary-path* (constantly binary))
   (apply require namespaces)
 
+  (llm-mock.server/start!)
+
   (let [timeout-minutes (if (re-find #"(?i)win|mac" (System/getProperty "os.name"))
                           10 ;; win and mac ci runs take longer
                           5)
         test-results (timeout (* timeout-minutes 60 1000)
                               #(with-log-tail-report
                                  (apply t/run-tests namespaces)))]
 
+    (llm-mock.server/stop!)
+
     (when (= test-results :timed-out)
       (println)
       (println (format "Timeout after %d minutes running integration tests!" timeout-minutes))

integration-test/integration/chat_openai_test.clj

@@ -3,37 +3,109 @@
    [clojure.test :refer [deftest is testing]]
    [integration.eca :as eca]
    [integration.fixture :as fixture]
+   [llm-mock.mocks :as llm.mocks]
    [matcher-combinators.matchers :as m]
    [matcher-combinators.test :refer [match?]]))
 
 (eca/clean-after-test)
 
+(defn match-content [chat-id request-id role content]
+  (is (match?
+       {:chatId chat-id
+        :requestId request-id
+        :role role
+        :content content}
+       (eca/client-awaits-server-notification :chat/contentReceived))))
+
 (deftest simple-text
   (eca/start-process!)
 
   (eca/request! (fixture/initialize-request))
   (eca/notify! (fixture/initialized-notification))
-  (testing "simple hello message with reply"
-    (let [resp (eca/request! (fixture/chat-prompt-request
-                              {:request-id 0
-                               :message "Hello there!"}))
-          chat-id (:chatId resp)]
-
-      (is (match?
-           {:chatId (m/pred string?)
-            :model "claude-sonnet-4-20250514"
-            :status "success"}
-           resp))
-
-      (is (match?
-           {:chatId chat-id
-            :requestId 0
-            :role "user"
-            :content {:type "text" :text "Hello there!\n"}}
-           (eca/client-awaits-server-notification :chat/contentReceived)))
-      (is (match?
-           {:chatId chat-id
-            :requestId 0
-            :role "system"
-            :content {:type "progress" :state "running" :text "Waiting model"}}
-           (eca/client-awaits-server-notification :chat/contentReceived))))))
+  (let [chat-id* (atom nil)]
+    (testing "We send a simple hello message"
+      (llm.mocks/set-case! :simple-text-0)
+      (let [req-id 0
+            resp (eca/request! (fixture/chat-prompt-request
+                                 {:request-id req-id
+                                  :model "gpt-5"
+                                  :message "Tell me a joke!"}))
+            chat-id (reset! chat-id* (:chatId resp))]
+
+        (is (match?
+              {:chatId (m/pred string?)
+               :model "gpt-5"
+               :status "success"}
+              resp))
+
+        (match-content chat-id req-id "user" {:type "text" :text "Tell me a joke!\n"})
+        (match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
+        (match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
+        (match-content chat-id req-id "assistant" {:type "text" :text "Knock"})
+        (match-content chat-id req-id "assistant" {:type "text" :text " knock!"})
+        (match-content chat-id req-id "system" {:type "usage"
+                                                :messageInputTokens 10
+                                                :messageOutputTokens 20
+                                                :sessionTokens 30
+                                                :messageCost (m/pred string?)
+                                                :sessionCost (m/pred string?)})
+        (match-content chat-id req-id "system" {:type "progress" :state "finished"})))
+
+    (testing "We reply"
+      (llm.mocks/set-case! :simple-text-1)
+      (let [req-id 1
+            resp (eca/request! (fixture/chat-prompt-request
+                                 {:chat-id @chat-id*
+                                  :request-id req-id
+                                  :model "gpt-5"
+                                  :message "Who's there?"}))
+            chat-id @chat-id*]
+
+        (is (match?
+              {:chatId (m/pred string?)
+               :model "gpt-5"
+               :status "success"}
+              resp))
+
+        (match-content chat-id req-id "user" {:type "text" :text "Who's there?\n"})
+        (match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
+        (match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
+        (match-content chat-id req-id "assistant" {:type "text" :text "Foo"})
+        (match-content chat-id req-id "system" {:type "usage"
+                                                :messageInputTokens 10
+                                                :messageOutputTokens 5
+                                                :sessionTokens 45
+                                                :messageCost (m/pred string?)
+                                                :sessionCost (m/pred string?)})
+        (match-content chat-id req-id "system" {:type "progress" :state "finished"})))
+
+    (testing "model reply again keeping context"
+      (llm.mocks/set-case! :simple-text-2)
+      (let [req-id 2
+            resp (eca/request! (fixture/chat-prompt-request
+                                 {:chat-id @chat-id*
+                                  :request-id req-id
+                                  :model "gpt-5"
+                                  :message "What foo?"}))
+            chat-id @chat-id*]
+
+        (is (match?
+              {:chatId (m/pred string?)
+               :model "gpt-5"
+               :status "success"}
+              resp))
+
+        (match-content chat-id req-id "user" {:type "text" :text "What foo?\n"})
+        (match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
+        (match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
+        (match-content chat-id req-id "assistant" {:type "text" :text "Foo"})
+        (match-content chat-id req-id "assistant" {:type "text" :text " bar!"})
+        (match-content chat-id req-id "assistant" {:type "text" :text "\n\n"})
+        (match-content chat-id req-id "assistant" {:type "text" :text "Ha!"})
+        (match-content chat-id req-id "system" {:type "usage"
+                                                :messageInputTokens 5
+                                                :messageOutputTokens 15
+                                                :sessionTokens 65
+                                                :messageCost (m/pred string?)
+                                                :sessionCost (m/pred string?)})
+        (match-content chat-id req-id "system" {:type "progress" :state "finished"})))))

integration-test/integration/fixture.clj

@@ -1,9 +1,15 @@
 (ns integration.fixture
   (:require
    [clojure.java.io :as io]
-   [integration.helper :as h]))
+   [integration.helper :as h]
+   [llm-mock.server :as llm-mock.server]))
 
-(def default-init-options {:pureConfig true})
+(def ^:private base-llm-mock-url
+  (str "http://localhost:" llm-mock.server/port))
+
+(def default-init-options {:pureConfig true
+                           :openaiApiUrl (str base-llm-mock-url "/openai")
+                           :anthropicApiUrl (str base-llm-mock-url "/anthropic")})
 
 (defn initialize-request
   ([]

integration-test/llm_mock/anthropic.clj

@@ -0,0 +1,39 @@
+(ns llm-mock.anthropic
+  (:require
+   [cheshire.core :as json]
+   [org.httpkit.server :as hk]))
+
+(defn ^:private sse-send!
+  "Send one SSE event line pair: event + data, followed by a blank line.
+  Keep the connection open until the scenario finishes."
+  [ch event m]
+  (hk/send! ch (str "event: " event "\n") false)
+  (hk/send! ch (str "data: " (json/generate-string m) "\n\n") false))
+
+(defn ^:private anthropic-simple-text! [ch]
+  ;; Stream minimal text and then finish with usage via message_delta
+  (sse-send! ch "content_block_delta"
+             {:type "content_block_delta"
+              :index 0
+              :delta {:type "text_delta" :text "Hello"}})
+  (sse-send! ch "content_block_delta"
+             {:type "content_block_delta"
+              :index 0
+              :delta {:type "text_delta" :text " world!"}})
+  (sse-send! ch "message_delta"
+             {:type "message_delta"
+              :delta {:stop_reason "end_turn"}
+              :usage {:input_tokens 10
+                      :output_tokens 3}})
+  (hk/close ch))
+
+(defn handle-anthropic-messages [req]
+  (hk/as-channel
+   req
+   {:on-open (fn [ch]
+               (hk/send! ch {:status 200
+                             :headers {"Content-Type" "text/event-stream; charset=utf-8"
+                                       "Cache-Control" "no-cache"
+                                       "Connection" "keep-alive"}}
+                         false)
+               (anthropic-simple-text! ch))}))

integration-test/llm_mock/mocks.clj

@@ -0,0 +1,6 @@
+(ns llm-mock.mocks)
+
+(def ^:dynamic *case* nil)
+
+(defn set-case! [case]
+  (alter-var-root #'*case* (constantly case)))

integration-test/llm_mock/openai.clj

@@ -0,0 +1,68 @@
+(ns llm-mock.openai
+  (:require
+   [cheshire.core :as json]
+   [llm-mock.mocks :as llm.mocks]
+   [org.httpkit.server :as hk]))
+
+(defn ^:private sse-send!
+  "Send one SSE event line pair: event + data, followed by a blank line.
+  Keep the connection open until the scenario finishes."
+  [ch event m]
+  (hk/send! ch (str "event: " event "\n") false)
+  (hk/send! ch (str "data: " (json/generate-string m) "\n\n") false))
+
+(defn ^:private simple-text-0 [ch]
+  (sse-send! ch "response.output_text.delta"
+             {:type "response.output_text.delta" :delta "Knock"})
+  (sse-send! ch "response.output_text.delta"
+             {:type "response.output_text.delta" :delta " knock!"})
+  (sse-send! ch "response.completed"
+             {:type "response.completed"
+              :response {:output []
+                         :usage {:input_tokens 10
+                                 :output_tokens 20}
+                         :status "completed"}})
+  (hk/close ch))
+
+(defn ^:private simple-text-1 [ch]
+  (sse-send! ch "response.output_text.delta"
+             {:type "response.output_text.delta" :delta "Foo"})
+  (sse-send! ch "response.completed"
+             {:type "response.completed"
+              :response {:output []
+                         :usage {:input_tokens 10
+                                 :output_tokens 5}
+                         :status "completed"}})
+  (hk/close ch))
+
+(defn ^:private simple-text-2 [ch]
+  (sse-send! ch "response.output_text.delta"
+             {:type "response.output_text.delta" :delta "Foo"})
+  (sse-send! ch "response.output_text.delta"
+             {:type "response.output_text.delta" :delta " bar!"})
+  (sse-send! ch "response.output_text.delta"
+             {:type "response.output_text.delta" :delta "\n\n"})
+  (sse-send! ch "response.output_text.delta"
+             {:type "response.output_text.delta" :delta "Ha!"})
+  (sse-send! ch "response.completed"
+             {:type "response.completed"
+              :response {:output []
+                         :usage {:input_tokens 5
+                                 :output_tokens 15}
+                         :status "completed"}})
+  (hk/close ch))
+
+(defn handle-openai-responses [req]
+  (hk/as-channel
+   req
+   {:on-open (fn [ch]
+                ;; initial SSE handshake
+               (hk/send! ch {:status 200
+                             :headers {"Content-Type" "text/event-stream; charset=utf-8"
+                                       "Cache-Control" "no-cache"
+                                       "Connection" "keep-alive"}}
+                         false)
+               (case llm.mocks/*case*
+                 :simple-text-0 (simple-text-0 ch)
+                 :simple-text-1 (simple-text-1 ch)
+                 :simple-text-2 (simple-text-2 ch)))}))

integration-test/llm_mock/server.clj

@@ -0,0 +1,34 @@
+(ns llm-mock.server
+  (:require
+   [llm-mock.anthropic :as llm-mock.anthropic]
+   [llm-mock.openai :as llm-mock.openai]
+   [org.httpkit.server :as hk]))
+
+(def port 6660)
+
+(defonce ^:private server* (atom nil))
+
+(defn ^:private app [req]
+  (let [{:keys [request-method uri]} req]
+    (cond
+      (and (= :post request-method)
+           (= uri "/openai/v1/responses"))
+      (llm-mock.openai/handle-openai-responses req)
+
+      (and (= :post request-method)
+           (= uri "/anthropic/v1/messages"))
+      (llm-mock.anthropic/handle-anthropic-messages req)
+
+      :else {:status 404 :headers {} :body "not found"})))
+
+(defn start! []
+  (when-not @server*
+    (println "Starting LLM mock server...")
+    (reset! server* (hk/run-server app {:port port})))
+  :started)
+
+(defn stop! []
+  (when-let [server @server*]
+    (server :timeout 100)
+    (reset! server* nil)
+    :stopped))

src/eca/config.clj

@@ -18,6 +18,8 @@
 (def initial-config
   {:openaiApiKey nil
    :anthropicApiKey nil
+   :openaiApiUrl nil
+   :anthropicApiUrl nil
    :rules []
    :commands []
    :nativeTools {:filesystem {:enabled true}