Fix flaky integration tests

ericdallo · ericdallo · commit 9d2b80a4147b · 2025-10-27T13:07:50.000-03:00
diff --git a/.cljfmt.edn b/.cljfmt.edn
@@ -1 +1,2 @@
-{:extra-indents {task [[:inner 0]]}}
+{:extra-indents {task [[:inner 0]]
+                 future* [[:inner 0]]}}
diff --git a/integration-test/integration/chat/anthropic_test.clj b/integration-test/integration/chat/anthropic_test.clj
@@ -30,6 +30,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "Tell me a joke!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "Knock"})
@@ -99,7 +100,6 @@
                                          :lastMessageCost (m/pred string?)
                                          :sessionCost (m/pred string?)})
         (match-content chat-id "system" {:type "progress" :state "finished"})
-        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (is (match?
              {:messages [{:role "user" :content [{:type "text" :text "Tell me a joke!"}]}
                          {:role "assistant" :content [{:type "text" :text "Knock knock!"}]}
@@ -128,6 +128,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "hello!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
@@ -141,7 +142,6 @@
                                          :lastMessageCost (m/pred string?)
                                          :sessionCost (m/pred string?)})
         (match-content chat-id "system" {:type "progress" :state "finished"})
-        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (is (match?
              {:messages [{:role "user" :content [{:type "text" :text "hello!"}]}]
               :system (m/pred vector?)}
@@ -206,6 +206,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "What files you see?\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
@@ -264,7 +265,6 @@
         (match-content chat-id "assistant" {:type "text" :text "The files I see:\n"})
         (match-content chat-id "assistant" {:type "text" :text "file1\nfile2\n"})
         (match-content chat-id "system" {:type "progress" :state "finished"})
-        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (is (match?
              {:messages [{:role "user" :content [{:type "text" :text "What files you see?"}]}
                          {:role "assistant"
diff --git a/integration-test/integration/chat/custom_provider_test.clj b/integration-test/integration/chat/custom_provider_test.clj
@@ -46,6 +46,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "Tell me a joke!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "Knock"})
@@ -158,6 +159,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "Tell me a joke!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "Knock "})
diff --git a/integration-test/integration/chat/github_copilot_test.clj b/integration-test/integration/chat/github_copilot_test.clj
@@ -31,6 +31,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "Tell me a joke!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "Knock "})
@@ -62,7 +63,6 @@
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "Foo"})
         (match-content chat-id "system" {:type "progress" :state "finished"})
-        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (is (match?
              {:input [{:role "user" :content [{:type "input_text" :text "Tell me a joke!"}]}
                       {:role "assistant" :content [{:type "output_text" :text "Knock knock!"}]}
@@ -84,7 +84,6 @@
               :status "prompting"}
              resp))
 
-        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "user" {:type "text" :text "What foo?\n"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
@@ -124,6 +123,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "hello!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
diff --git a/integration-test/integration/chat/google_test.clj b/integration-test/integration/chat/google_test.clj
@@ -31,6 +31,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "Tell me a joke!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "Knoc"})
@@ -62,7 +63,6 @@
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "Foo"})
         (match-content chat-id "system" {:type "progress" :state "finished"})
-        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (is (match?
              {:input [{:role "user" :content [{:type "input_text" :text "Tell me a joke!"}]}
                       {:role "assistant" :content [{:type "output_text" :text "Knock knock!"}]}
@@ -84,7 +84,6 @@
               :status "prompting"}
              resp))
 
-        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "user" {:type "text" :text "What foo?\n"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
@@ -123,6 +122,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "hello!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
diff --git a/integration-test/integration/chat/ollama_test.clj b/integration-test/integration/chat/ollama_test.clj
@@ -38,6 +38,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "Tell me a joke!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "Knock"})
@@ -129,6 +130,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "hello!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
@@ -205,6 +207,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "What files you see?\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "I will list files"})
diff --git a/integration-test/integration/chat/openai_test.clj b/integration-test/integration/chat/openai_test.clj
@@ -30,6 +30,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "Tell me a joke!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "text" :text "Knock"})
@@ -39,7 +40,6 @@
                                          :lastMessageCost (m/pred string?)
                                          :sessionCost (m/pred string?)})
         (match-content chat-id "system" {:type "progress" :state "finished"})
-        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (is (match?
              {:input [{:role "user" :content [{:type "input_text" :text "Tell me a joke!"}]}]
               :instructions (m/pred string?)}
@@ -128,6 +128,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "hello!\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "reasonStarted" :id "123"})
@@ -205,6 +206,7 @@
              resp))
 
         (match-content chat-id "user" {:type "text" :text "What files you see?\n"})
+        (match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
         (match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
         (match-content chat-id "assistant" {:type "reasonStarted" :id "123"})
diff --git a/integration-test/integration/fixture.clj b/integration-test/integration/fixture.clj
@@ -22,6 +22,7 @@
                      :keyEnv "FOO"}})
 
 (def default-init-options {:pureConfig true
+                           :env "test"
                            :toolCall {:approval {:byDefault "allow"}}
                            :providers default-providers})
 
diff --git a/integration-test/llm_mock/anthropic.clj b/integration-test/llm_mock/anthropic.clj
@@ -224,9 +224,7 @@
                                          "Connection" "keep-alive"}}
                            false)
                  (if (string/includes? (:text (last (:system body))) llm.mocks/chat-title-generator-str)
-                   (do
-                     (Thread/sleep 2000) ;; avoid tests failing with mismatch order of contents
-                     (chat-title-text-0 ch))
+                   (chat-title-text-0 ch)
                    (do
                      (llm.mocks/set-req-body! llm.mocks/*case* body)
                      (case llm.mocks/*case*
diff --git a/integration-test/llm_mock/ollama.clj b/integration-test/llm_mock/ollama.clj
@@ -81,9 +81,7 @@
                                          "Connection" "keep-alive"}}
                            false)
                  (if (string/includes? (:content (first (:messages body))) llm.mocks/chat-title-generator-str)
-                   (do
-                     (Thread/sleep 2000) ;; avoid tests failing with mismatch order of contents
-                     (chat-title-text-0 ch))
+                   (chat-title-text-0 ch)
                    (do
                      (llm.mocks/set-req-body! llm.mocks/*case* body)
                      (case llm.mocks/*case*
diff --git a/integration-test/llm_mock/openai.clj b/integration-test/llm_mock/openai.clj
@@ -198,10 +198,7 @@
                                          "Connection" "keep-alive"}}
                            false)
                  (if (string/includes? (:instructions body) llm.mocks/chat-title-generator-str)
-                   (do
-                     ;; TODO improve this
-                     (Thread/sleep 4000) ;; avoid tests failing with mismatch order of contents
-                     (chat-title-text-0 ch))
+                   (chat-title-text-0 ch)
                    (do
                      (llm.mocks/set-req-body! llm.mocks/*case* body)
                      (case llm.mocks/*case*
diff --git a/integration-test/llm_mock/openai_chat.clj b/integration-test/llm_mock/openai_chat.clj
@@ -111,9 +111,7 @@
                                          "Connection" "keep-alive"}}
                            false)
                  (if (string/includes? (:content (first (:messages body))) llm.mocks/chat-title-generator-str)
-                   (do
-                     (Thread/sleep 3000) ;; avoid tests failing with mismatch order of contents
-                     (chat-title-text-0 ch))
+                   (chat-title-text-0 ch)
                    (do
                      (llm.mocks/set-req-body! llm.mocks/*case* normalized-body)
                      (case llm.mocks/*case*
diff --git a/src/eca/config.clj b/src/eca/config.clj
@@ -120,7 +120,8 @@
            :repoMap {:maxTotalEntries 800
                      :maxEntriesPerDir 50}}
    :completion {:model "openai/gpt-4.1"
-                :systemPromptFile "prompts/inline_completion.md"}})
+                :systemPromptFile "prompts/inline_completion.md"}
+   :env "prod"})
    
 
 (def ^:private fallback-behavior "agent")
diff --git a/src/eca/features/chat.clj b/src/eca/features/chat.clj
@@ -18,7 +18,7 @@
    [eca.logger :as logger]
    [eca.messenger :as messenger]
    [eca.metrics :as metrics]
-   [eca.shared :as shared :refer [assoc-some]]))
+   [eca.shared :as shared :refer [assoc-some future*]]))
 
 (set! *warn-on-reflection* true)
 
@@ -560,7 +560,7 @@
                                                    usage))))]
 
     (when-not (get-in db [:chats chat-id :title])
-      (future
+      (future* config
         (when-let [{:keys [result]} (llm-api/sync-prompt!
                                      {:provider provider
                                       :model model
diff --git a/src/eca/shared.clj b/src/eca/shared.clj
@@ -185,3 +185,10 @@
     (.format (.atZoneSameInstant (.atOffset (Instant/ofEpochMilli ms) ZoneOffset/UTC)
                                  (ZoneId/systemDefault))
              (DateTimeFormatter/ofPattern pattern))))
+
+(defmacro future*
+  "Wrapper for future unless in tests"
+  [config & body]
+  `(if (= "test" (:env ~config))
+     ~@body
+     (future ~@body)))

Original file line number	Diff line number	Diff line change
`@@ -1 +1,2 @@`
`1`		`-{:extra-indents {task [[:inner 0]]}}`
	`1`	`+{:extra-indents {task [[:inner 0]]`
	`2`	`+ future* [[:inner 0]]}}`