Skip to content

Commit 9a003a3

Browse files
ericdalloericdallo
committed
Add google integration-tests
1 parent e70780a commit 9a003a3

File tree

4 files changed

+300
-0
lines changed

4 files changed

+300
-0
lines changed

‎integration-test/entrypoint.clj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@
1010
integration.chat.openai-test
1111
integration.chat.anthropic-test
1212
integration.chat.github-copilot-test
13+
integration.chat.google-test
1314
integration.chat.ollama-test
1415
integration.chat.custom-provider-test
1516
integration.chat.commands-test

‎integration-test/integration/chat/google_test.clj‎

Lines changed: 292 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,292 @@
1+
(ns integration.chat.google-test
2+
(:require
3+
[clojure.test :refer [deftest is testing]]
4+
[integration.eca :as eca]
5+
[integration.fixture :as fixture]
6+
[integration.helper :refer [match-content] :as h]
7+
[llm-mock.mocks :as llm.mocks]
8+
[matcher-combinators.matchers :as m]
9+
[matcher-combinators.test :refer [match?]]))
10+
11+
(eca/clean-after-test)
12+
13+
(deftest simple-text
14+
(eca/start-process!)
15+
16+
(eca/request! (fixture/initialize-request))
17+
(eca/notify! (fixture/initialized-notification))
18+
(let [chat-id* (atom nil)]
19+
(testing "We send a simple hello message"
20+
(llm.mocks/set-case! :simple-text-0)
21+
(let [resp (eca/request! (fixture/chat-prompt-request
22+
{:model "google/gemini-2.5-pro"
23+
:message "Tell me a joke!"}))
24+
chat-id (reset! chat-id* (:chatId resp))]
25+
26+
(is (match?
27+
{:chatId (m/pred string?)
28+
:model "google/gemini-2.5-pro"
29+
:status "prompting"}
30+
resp))
31+
32+
(match-content chat-id "user" {:type "text" :text "Tell me a joke!\n"})
33+
(match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
34+
(match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
35+
(match-content chat-id "assistant" {:type "text" :text "Knock"})
36+
(match-content chat-id "assistant" {:type "text" :text " knock!"})
37+
(match-content chat-id "system" {:type "progress" :state "finished"})
38+
(is (match?
39+
{:input [{:role "user" :content [{:type "input_text" :text "Tell me a joke!"}]}]
40+
:instructions (m/pred string?)}
41+
(llm.mocks/get-req-body :simple-text-0)))))
42+
43+
(testing "We reply"
44+
(llm.mocks/set-case! :simple-text-1)
45+
(let [resp (eca/request! (fixture/chat-prompt-request
46+
{:chat-id @chat-id*
47+
:model "google/gemini-2.5-pro"
48+
:message "Who's there?"}))
49+
chat-id @chat-id*]
50+
51+
(is (match?
52+
{:chatId (m/pred string?)
53+
:model "google/gemini-2.5-pro"
54+
:status "prompting"}
55+
resp))
56+
57+
(match-content chat-id "user" {:type "text" :text "Who's there?\n"})
58+
(match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
59+
(match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
60+
(match-content chat-id "assistant" {:type "text" :text "Foo"})
61+
(match-content chat-id "system" {:type "progress" :state "finished"})
62+
(match-content chat-id "system" {:type "metadata" :title "Some Cool Title"})
63+
(is (match?
64+
{:input [{:role "user" :content [{:type "input_text" :text "Tell me a joke!"}]}
65+
{:role "assistant" :content [{:type "output_text" :text "Knock knock!"}]}
66+
{:role "user" :content [{:type "input_text" :text "Who's there?"}]}]
67+
:instructions (m/pred string?)}
68+
(llm.mocks/get-req-body :simple-text-1)))))
69+
70+
(testing "model reply again keeping context"
71+
(llm.mocks/set-case! :simple-text-2)
72+
(let [resp (eca/request! (fixture/chat-prompt-request
73+
{:chat-id @chat-id*
74+
:model "google/gemini-2.5-pro"
75+
:message "What foo?"}))
76+
chat-id @chat-id*]
77+
78+
(is (match?
79+
{:chatId (m/pred string?)
80+
:model "google/gemini-2.5-pro"
81+
:status "prompting"}
82+
resp))
83+
84+
(match-content chat-id "user" {:type "text" :text "What foo?\n"})
85+
(match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
86+
(match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
87+
(match-content chat-id "assistant" {:type "text" :text "Foo"})
88+
(match-content chat-id "assistant" {:type "text" :text " bar!"})
89+
(match-content chat-id "assistant" {:type "text" :text "\n\n"})
90+
(match-content chat-id "assistant" {:type "text" :text "Ha!"})
91+
(match-content chat-id "system" {:type "progress" :state "finished"})
92+
(is (match?
93+
{:input [{:role "user" :content [{:type "input_text" :text "Tell me a joke!"}]}
94+
{:role "assistant" :content [{:type "output_text" :text "Knock knock!"}]}
95+
{:role "user" :content [{:type "input_text" :text "Who's there?"}]}
96+
{:role "assistant" :content [{:type "output_text" :text "Foo"}]}
97+
{:role "user" :content [{:type "input_text" :text "What foo?"}]}]
98+
:instructions (m/pred string?)}
99+
(llm.mocks/get-req-body :simple-text-2)))))))
100+
101+
;; TODO fix reasoning in openai-chat
102+
#_(deftest reasoning-text
103+
(eca/start-process!)
104+
105+
(eca/request! (fixture/initialize-request))
106+
(eca/notify! (fixture/initialized-notification))
107+
(let [chat-id* (atom nil)]
108+
(testing "We send a hello message"
109+
(llm.mocks/set-case! :reasoning-0)
110+
(let [resp (eca/request! (fixture/chat-prompt-request
111+
{:model "google/gemini-2.5-pro"
112+
:message "hello!"}))
113+
chat-id (reset! chat-id* (:chatId resp))]
114+
115+
(is (match?
116+
{:chatId (m/pred string?)
117+
:model "google/gemini-2.5-pro"
118+
:status "prompting"}
119+
resp))
120+
121+
(match-content chat-id "user" {:type "text" :text "hello!\n"})
122+
(match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
123+
(match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
124+
(match-content chat-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
125+
(match-content chat-id "assistant" {:type "reasonText" :id (m/pred string?) :text "I should say"})
126+
(match-content chat-id "assistant" {:type "reasonText" :id (m/pred string?) :text " hello"})
127+
(match-content chat-id "assistant" {:type "reasonFinished" :id (m/pred string?) :totalTimeMs (m/pred number?)})
128+
(match-content chat-id "assistant" {:type "text" :text "hello"})
129+
(match-content chat-id "assistant" {:type "text" :text " there!"})
130+
(match-content chat-id "system" {:type "usage"
131+
:messageInputTokens 5
132+
:messageOutputTokens 30
133+
:sessionTokens 35
134+
:messageCost (m/pred string?)
135+
:sessionCost (m/pred string?)})
136+
(match-content chat-id "system" {:type "progress" :state "finished"})
137+
(is (match?
138+
{:input [{:role "user" :content [{:type "input_text" :text "hello"}]}]
139+
:instructions (m/pred string?)}
140+
llm.mocks/*last-req-body*))))
141+
142+
(testing "We reply"
143+
(llm.mocks/set-case! :reasoning-1)
144+
(let [resp (eca/request! (fixture/chat-prompt-request
145+
{:chat-id @chat-id*
146+
:model "google/gemini-2.5-pro"
147+
:message "how are you?"}))
148+
chat-id @chat-id*]
149+
150+
(is (match?
151+
{:chatId (m/pred string?)
152+
:model "google/gemini-2.5-pro"
153+
:status "prompting"}
154+
resp))
155+
156+
(match-content chat-id "user" {:type "text" :text "how are you?\n"})
157+
(match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
158+
(match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
159+
(match-content chat-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
160+
(match-content chat-id "assistant" {:type "reasonText" :id (m/pred string?) :text "I should say"})
161+
(match-content chat-id "assistant" {:type "reasonText" :id (m/pred string?) :text " fine"})
162+
(match-content chat-id "assistant" {:type "reasonFinished" :id (m/pred string?) :totalTimeMs (m/pred number?)})
163+
(match-content chat-id "assistant" {:type "text" :text "I'm "})
164+
(match-content chat-id "assistant" {:type "text" :text " fine"})
165+
(match-content chat-id "system" {:type "usage"
166+
:messageInputTokens 10
167+
:messageOutputTokens 20
168+
:sessionTokens 65
169+
:messageCost (m/pred string?)
170+
:sessionCost (m/pred string?)})
171+
(match-content chat-id "system" {:type "progress" :state "finished"})
172+
(is (match?
173+
{:input [{:role "user" :content [{:type "input_text" :text "hello!"}]}]
174+
:instructions (m/pred string?)}
175+
{:messages [{:role "user" :content [{:type "text" :text "hello!"}]}
176+
{:role "assistant"
177+
:content [{:type "thinking"
178+
:signature "enc-123"
179+
:thinking "I should say hello"}]}
180+
{:role "assistant" :content [{:type "text" :text "hello there!"}]}
181+
{:role "user" :content [{:type "text" :text "how are you?"}]}]
182+
:system (m/pred vector?)}
183+
llm.mocks/*last-req-body*))))))
184+
185+
#_(deftest tool-calling
186+
(eca/start-process!)
187+
188+
(eca/request! (fixture/initialize-request))
189+
(eca/notify! (fixture/initialized-notification))
190+
(let [chat-id* (atom nil)]
191+
(testing "We ask what files LLM see"
192+
(llm.mocks/set-case! :tool-calling-0)
193+
(let [0
194+
resp (eca/request! (fixture/chat-prompt-request
195+
{:model "google/gemini-2.5-pro"
196+
:message "What files you see?"}))
197+
chat-id (reset! chat-id* (:chatId resp))]
198+
199+
(is (match?
200+
{:chatId (m/pred string?)
201+
:model "google/gemini-2.5-pro"
202+
:status "prompting"}
203+
resp))
204+
205+
(match-content chat-id "user" {:type "text" :text "What files you see?\n"})
206+
(match-content chat-id "system" {:type "progress" :state "running" :text "Waiting model"})
207+
(match-content chat-id "system" {:type "progress" :state "running" :text "Generating"})
208+
(match-content chat-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
209+
(match-content chat-id "assistant" {:type "reasonText" :id (m/pred string?) :text "I should call tool"})
210+
(match-content chat-id "assistant" {:type "reasonText" :id (m/pred string?) :text " eca_directory_tree"})
211+
(match-content chat-id "assistant" {:type "reasonFinished" :id (m/pred string?) :totalTimeMs (m/pred number?)})
212+
(match-content chat-id "assistant" {:type "text" :text "I will list files"})
213+
(match-content chat-id "assistant" {:type "toolCallPrepare"
214+
:origin "native"
215+
:id "tool-1"
216+
:name "eca_directory_tree"
217+
:argumentsText ""
218+
:manualApproval false
219+
:summary "Listing file tree"})
220+
(match-content chat-id "assistant" {:type "toolCallPrepare"
221+
:origin "native"
222+
:id "tool-1"
223+
:name "eca_directory_tree"
224+
:argumentsText "{\"pat"
225+
:manualApproval false
226+
:summary "Listing file tree"})
227+
(match-content chat-id "assistant" {:type "toolCallPrepare"
228+
:origin "native"
229+
:id "tool-1"
230+
:name "eca_directory_tree"
231+
:argumentsText (str "h\":\"" (h/project-path->canon-path "resources") "\"}")
232+
:manualApproval false
233+
:summary "Listing file tree"})
234+
(match-content chat-id "system" {:type "usage"
235+
:messageInputTokens 5
236+
:messageOutputTokens 30
237+
:sessionTokens 35
238+
:messageCost (m/pred string?)
239+
:sessionCost (m/pred string?)})
240+
(match-content chat-id "assistant" {:type "toolCallRun"
241+
:origin "native"
242+
:id "tool-1"
243+
:name "eca_directory_tree"
244+
:arguments {:path (h/project-path->canon-path "resources")}
245+
:manualApproval false
246+
:summary "Listing file tree"})
247+
(match-content chat-id "assistant" {:type "toolCallRunning"
248+
:origin "native"
249+
:id "tool-1"
250+
:name "eca_directory_tree"
251+
:arguments {:path (h/project-path->canon-path "resources")}
252+
:totalTimeMs number?
253+
:summary "Listing file tree"})
254+
(match-content chat-id "assistant" {:type "toolCalled"
255+
:origin "native"
256+
:id "tool-1"
257+
:name "eca_directory_tree"
258+
:arguments {:path (h/project-path->canon-path "resources")}
259+
:summary "Listing file tree"
260+
:error false
261+
:outputs [{:type "text" :text (str "[FILE] " (h/project-path->canon-path "resources/file1.md\n")
262+
"[FILE] " (h/project-path->canon-path "resources/file2.md\n"))}]})
263+
(match-content chat-id "assistant" {:type "text" :text "The files I see:\n"})
264+
(match-content chat-id "assistant" {:type "text" :text "file1\nfile2\n"})
265+
(match-content chat-id "system" {:type "usage"
266+
:messageInputTokens 5
267+
:messageOutputTokens 30
268+
:sessionTokens 70
269+
:messageCost (m/pred string?)
270+
:sessionCost (m/pred string?)})
271+
(match-content chat-id "system" {:type "progress" :state "finished"})
272+
(is (match?
273+
{:messages [{:role "user" :content [{:type "text" :text "What files you see?"}]}
274+
{:role "assistant"
275+
:content [{:type "thinking"
276+
:signature "enc-123"
277+
:thinking "I should call tool eca_directory_tree"}]}
278+
{:role "assistant" :content [{:type "text" :text "I will list files"}]}
279+
{:role "assistant"
280+
:content [{:type "tool_use"
281+
:id "tool-1"
282+
:name "eca_directory_tree"
283+
:input {:path (h/project-path->canon-path "resources")}}]}
284+
{:role "user"
285+
:content [{:type "tool_result"
286+
:tool_use_id "tool-1"
287+
:content (str "[FILE] " (h/project-path->canon-path "resources/file1.md\n")
288+
"[FILE] " (h/project-path->canon-path "resources/file2.md\n\n"))}]}]
289+
:tools (m/embeds
290+
[{:name "eca_directory_tree"}])
291+
:system (m/pred vector?)}
292+
llm.mocks/*last-req-body*))))))

‎integration-test/integration/fixture.clj‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,9 @@
1515
:key "foo-key"
1616
:keyEnv "FOO"}
1717
"github-copilot" {:url (str base-llm-mock-url "/github-copilot")
18+
:key "foo-key"
19+
:keyEnv "FOO"}
20+
"google" {:url (str base-llm-mock-url "/google")
1821
:key "foo-key"
1922
:keyEnv "FOO"}})
2023

‎integration-test/llm_mock/server.clj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,10 @@
2525
(= uri "/github-copilot/chat/completions"))
2626
(llm-mock.openai-chat/handle-openai-chat req)
2727

28+
(and (= :post request-method)
29+
(= uri "/google/chat/completions"))
30+
(llm-mock.openai-chat/handle-openai-chat req)
31+
2832
(and (= :post request-method)
2933
(= uri "/anthropic/v1/messages"))
3034
(llm-mock.anthropic/handle-anthropic-messages req)

0 commit comments

Comments
 (0)