Skip to content

Commit 485f4d6

Browse files
ericdalloericdallo
committed
tests: add integration tests for ollama
1 parent c11af56 commit 485f4d6

File tree

11 files changed

+393
-34
lines changed

11 files changed

+393
-34
lines changed

‎CHANGELOG.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22

33
## Unreleased
44

5+
- breaking: Replace configs `ollama host` and `ollama port` with `ollamaApiUrl`.
6+
57
## 0.26.2
68

79
- Fix `chat/queryContext` to not return already added contexts

‎docs/configuration.md‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -195,9 +195,10 @@ Example:
195195
```typescript
196196
interface Config {
197197
openaiApiKey?: string;
198-
anthropicApiKey?: string;
199198
openaiApiUrl?: string;
199+
anthropicApiKey?: string;
200200
anthropicApiUrl?: string;
201+
ollamaApiUrl: string;
201202
rules: [{path: string;}];
202203
commands: [{path: string;}];
203204
systemPromptTemplateFile?: string;
@@ -251,9 +252,10 @@ interface Config {
251252
```javascript
252253
{
253254
"openaiApiKey" : null,
254-
"anthropicApiKey" : null,
255255
"openaiApiUrl" : null,
256+
"anthropicApiKey" : null,
256257
"anthropicApiUrl" : null,
258+
"ollamaApiUrl": "http://localhost:11434"
257259
"rules" : [],
258260
"commands" : [],
259261
"nativeTools": {"filesystem": {"enabled": true},
@@ -268,8 +270,6 @@ interface Config {
268270
"customProviders": {},
269271
"models": {},
270272
"ollama" : {
271-
"host" : "http://localhost",
272-
"port" : 11434,
273273
"useTools": true,
274274
"think": true
275275
},

‎integration-test/entrypoint.clj‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,7 +7,8 @@
77
(def namespaces
88
'[integration.initialize-test
99
integration.chat.openai-test
10-
integration.chat.anthropic-test])
10+
integration.chat.anthropic-test
11+
integration.chat.ollama-test])
1112

1213
(defn timeout [timeout-ms callback]
1314
(let [fut (future (callback))

‎integration-test/integration/chat/ollama_test.clj‎

Lines changed: 250 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,250 @@
1+
(ns integration.chat.ollama-test
2+
(:require
3+
[clojure.test :refer [deftest is testing]]
4+
[integration.eca :as eca]
5+
[integration.fixture :as fixture]
6+
[integration.helper :refer [match-content] :as h]
7+
[llm-mock.mocks :as llm.mocks]
8+
[matcher-combinators.matchers :as m]
9+
[matcher-combinators.test :refer [match?]]))
10+
11+
(eca/clean-after-test)
12+
13+
(deftest simple-text
14+
(eca/start-process!)
15+
16+
(is (match?
17+
{:models (m/embeds ["ollama/qwen3"])}
18+
(eca/request! (fixture/initialize-request {:initializationOptions (merge fixture/default-init-options
19+
{:ollamaApiUrl (str fixture/base-llm-mock-url "/ollama")})
20+
:capabilities {:codeAssistant {:chat {}}}}))))
21+
(eca/notify! (fixture/initialized-notification))
22+
(let [chat-id* (atom nil)]
23+
(testing "We send a simple hello message"
24+
(llm.mocks/set-case! :simple-text-0)
25+
(let [req-id 0
26+
resp (eca/request! (fixture/chat-prompt-request
27+
{:request-id req-id
28+
:model "ollama/qwen3"
29+
:message "Tell me a joke!"}))
30+
chat-id (reset! chat-id* (:chatId resp))]
31+
32+
(is (match?
33+
{:chatId (m/pred string?)
34+
:model "ollama/qwen3"
35+
:status "success"}
36+
resp))
37+
38+
(match-content chat-id req-id "user" {:type "text" :text "Tell me a joke!\n"})
39+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
40+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
41+
(match-content chat-id req-id "assistant" {:type "text" :text "Knock"})
42+
(match-content chat-id req-id "assistant" {:type "text" :text " knock!"})
43+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
44+
(is (match?
45+
{:model "qwen3"
46+
:messages [{:role "system" :content (m/pred string?)}
47+
{:role "user" :content [{:type "text" :text "Tell me a joke!"}]}]}
48+
llm.mocks/*last-req-body*))))
49+
50+
(testing "We reply"
51+
(llm.mocks/set-case! :simple-text-1)
52+
(let [req-id 1
53+
resp (eca/request! (fixture/chat-prompt-request
54+
{:chat-id @chat-id*
55+
:request-id req-id
56+
:model "ollama/qwen3"
57+
:message "Who's there?"}))
58+
chat-id @chat-id*]
59+
60+
(is (match?
61+
{:chatId (m/pred string?)
62+
:model "ollama/qwen3"
63+
:status "success"}
64+
resp))
65+
66+
(match-content chat-id req-id "user" {:type "text" :text "Who's there?\n"})
67+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
68+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
69+
(match-content chat-id req-id "assistant" {:type "text" :text "Foo"})
70+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})))
71+
72+
(testing "model reply again keeping context"
73+
(llm.mocks/set-case! :simple-text-2)
74+
(let [req-id 2
75+
resp (eca/request! (fixture/chat-prompt-request
76+
{:chat-id @chat-id*
77+
:request-id req-id
78+
:model "ollama/qwen3"
79+
:message "What foo?"}))
80+
chat-id @chat-id*]
81+
82+
(is (match?
83+
{:chatId (m/pred string?)
84+
:model "ollama/qwen3"
85+
:status "success"}
86+
resp))
87+
88+
(match-content chat-id req-id "user" {:type "text" :text "What foo?\n"})
89+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
90+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
91+
(match-content chat-id req-id "assistant" {:type "text" :text "Foo"})
92+
(match-content chat-id req-id "assistant" {:type "text" :text " bar!"})
93+
(match-content chat-id req-id "assistant" {:type "text" :text "\n\n"})
94+
(match-content chat-id req-id "assistant" {:type "text" :text "Ha!"})
95+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
96+
(is (match?
97+
{:model "qwen3"
98+
:messages [{:role "system" :content (m/pred string?)}
99+
{:role "user" :content [{:type "text" :text "Tell me a joke!"}]}
100+
{:role "assistant" :content [{:type "text" :text "Knock knock!"}]}
101+
{:role "user" :content [{:type "text" :text "Who's there?"}]}
102+
{:role "assistant" :content [{:type "text" :text "Foo"}]}
103+
{:role "user" :content [{:type "text" :text "What foo?"}]}]}
104+
llm.mocks/*last-req-body*))))))
105+
106+
(deftest reasoning-text
107+
(eca/start-process!)
108+
109+
(is (match?
110+
{:models (m/embeds ["ollama/qwen3"])}
111+
(eca/request! (fixture/initialize-request {:initializationOptions (merge fixture/default-init-options
112+
{:ollamaApiUrl (str fixture/base-llm-mock-url "/ollama")})
113+
:capabilities {:codeAssistant {:chat {}}}}))))
114+
(eca/notify! (fixture/initialized-notification))
115+
(let [chat-id* (atom nil)]
116+
(testing "We send a hello message"
117+
(llm.mocks/set-case! :reasoning-0)
118+
(let [req-id 0
119+
resp (eca/request! (fixture/chat-prompt-request
120+
{:request-id req-id
121+
:model "ollama/qwen3"
122+
:message "hello!"}))
123+
chat-id (reset! chat-id* (:chatId resp))]
124+
125+
(is (match?
126+
{:chatId (m/pred string?)
127+
:model "ollama/qwen3"
128+
:status "success"}
129+
resp))
130+
131+
(match-content chat-id req-id "user" {:type "text" :text "hello!\n"})
132+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
133+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
134+
(match-content chat-id req-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
135+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text "I should say"})
136+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text " hello"})
137+
(match-content chat-id req-id "assistant" {:type "reasonFinished" :id (m/pred string?)})
138+
(match-content chat-id req-id "assistant" {:type "text" :text "hello"})
139+
(match-content chat-id req-id "assistant" {:type "text" :text " there!"})
140+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
141+
(is (match?
142+
{:model "qwen3"
143+
:messages [{:role "system" :content (m/pred string?)}
144+
{:role "user" :content [{:type "text" :text "hello!"}]}]}
145+
llm.mocks/*last-req-body*))))
146+
147+
(testing "We reply"
148+
(llm.mocks/set-case! :reasoning-1)
149+
(let [req-id 1
150+
resp (eca/request! (fixture/chat-prompt-request
151+
{:request-id req-id
152+
:chat-id @chat-id*
153+
:model "ollama/qwen3"
154+
:message "how are you?"}))
155+
chat-id @chat-id*]
156+
157+
(is (match?
158+
{:chatId (m/pred string?)
159+
:model "ollama/qwen3"
160+
:status "success"}
161+
resp))
162+
163+
(match-content chat-id req-id "user" {:type "text" :text "how are you?\n"})
164+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
165+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
166+
(match-content chat-id req-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
167+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text "I should say"})
168+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text " fine"})
169+
(match-content chat-id req-id "assistant" {:type "reasonFinished" :id (m/pred string?)})
170+
(match-content chat-id req-id "assistant" {:type "text" :text "I'm "})
171+
(match-content chat-id req-id "assistant" {:type "text" :text " fine"})
172+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
173+
(is (match?
174+
{:model "qwen3"
175+
:messages [{:role "system" :content (m/pred string?)}
176+
{:role "user" :content [{:type "text" :text "hello!"}]}
177+
{:role "assistant" :content "I should say hello"}
178+
{:role "assistant" :content [{:type "text" :text "hello there!"}]}
179+
{:role "user" :content [{:type "text" :text "how are you?"}]}]}
180+
llm.mocks/*last-req-body*))))))
181+
182+
(deftest tool-calling
183+
(eca/start-process!)
184+
185+
(is (match?
186+
{:models (m/embeds ["ollama/qwen3"])}
187+
(eca/request! (fixture/initialize-request {:initializationOptions (merge fixture/default-init-options
188+
{:ollamaApiUrl (str fixture/base-llm-mock-url "/ollama")})
189+
:capabilities {:codeAssistant {:chat {}}}}))))
190+
(eca/notify! (fixture/initialized-notification))
191+
(let [chat-id* (atom nil)]
192+
(testing "We ask what files LLM see"
193+
(llm.mocks/set-case! :tool-calling-0)
194+
(let [req-id 0
195+
resp (eca/request! (fixture/chat-prompt-request
196+
{:request-id req-id
197+
:model "ollama/qwen3"
198+
:message "What files you see?"}))
199+
chat-id (reset! chat-id* (:chatId resp))]
200+
201+
(is (match?
202+
{:chatId (m/pred string?)
203+
:model "ollama/qwen3"
204+
:status "success"}
205+
resp))
206+
207+
(match-content chat-id req-id "user" {:type "text" :text "What files you see?\n"})
208+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
209+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
210+
(match-content chat-id req-id "assistant" {:type "text" :text "I will list files"})
211+
(match-content chat-id req-id "assistant" {:type "toolCallPrepare"
212+
:origin "native"
213+
:id (m/pred string?)
214+
:name "eca_directory_tree"
215+
:argumentsText ""
216+
:manualApproval false
217+
:summary "Listing file tree"})
218+
(match-content chat-id req-id "assistant" {:type "toolCallRun"
219+
:origin "native"
220+
:id (m/pred string?)
221+
:name "eca_directory_tree"
222+
:arguments {:path (h/project-path->canon-path "resources")}
223+
:manualApproval false
224+
:summary "Listing file tree"})
225+
(match-content chat-id req-id "assistant" {:type "toolCalled"
226+
:origin "native"
227+
:id (m/pred string?)
228+
:name "eca_directory_tree"
229+
:arguments {:path (h/project-path->canon-path "resources")}
230+
:summary "Listing file tree"
231+
:error false
232+
:outputs [{:type "text" :text (str "[FILE] " (h/project-path->canon-path "resources/file1.md\n")
233+
"[FILE] " (h/project-path->canon-path "resources/file2.md\n"))}]})
234+
(match-content chat-id req-id "assistant" {:type "text" :text "The files I see:\n"})
235+
(match-content chat-id req-id "assistant" {:type "text" :text "file1\nfile2\n"})
236+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
237+
(is (match?
238+
{:model "qwen3"
239+
:messages [{:role "user" :content [{:type "text" :text "What files you see?"}]}
240+
{:role "assistant" :content [{:type "text" :text "I will list files"}]}
241+
{:role "assistant" :tool-calls [{:type "function"
242+
:function {:id (m/pred string?)
243+
:name "eca_directory_tree"
244+
:arguments {:path (h/project-path->canon-path "resources")}
245+
:summary "Listing file tree"
246+
:origin "native"}}]}
247+
{:role "tool" :content (str "[FILE] " (h/project-path->canon-path "resources/file1.md\n")
248+
"[FILE] " (h/project-path->canon-path "resources/file2.md\n\n"))}]
249+
:tools (m/embeds [{:type "function" :function {:name "eca_directory_tree"}}])}
250+
llm.mocks/*last-req-body*))))))

‎integration-test/integration/fixture.clj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
[integration.helper :as h]
55
[llm-mock.server :as llm-mock.server]))
66

7-
(def ^:private base-llm-mock-url
7+
(def base-llm-mock-url
88
(str "http://localhost:" llm-mock.server/port))
99

1010
(def default-init-options {:pureConfig true

‎integration-test/llm_mock/anthropic.clj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -140,7 +140,7 @@
140140
(some #(= "tool_result" (:type %)) content))
141141
(:messages body))]
142142
(if-not second-stage?
143-
(let [args-json (json/generate-string {:path (h/project-path->canon-path "resources")})]
143+
(do
144144
;; Thinking prelude
145145
(sse-send! ch "content_block_start"
146146
{:type "content_block_start"

0 commit comments

Comments
 (0)