Skip to content

Commit 6b9c793

Browse files
ericdalloericdallo
committed
Add tests for anthropic
1 parent 8b82bf1 commit 6b9c793

File tree

3 files changed

+503
-8
lines changed

3 files changed

+503
-8
lines changed

‎integration-test/entrypoint.clj‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,9 @@
55
[llm-mock.server :as llm-mock.server]))
66

77
(def namespaces
8-
'[;integration.initialize-test
9-
integration.chat.openai-test])
8+
'[integration.initialize-test
9+
integration.chat.openai-test
10+
integration.chat.anthropic-test])
1011

1112
(defn timeout [timeout-ms callback]
1213
(let [fut (future (callback))

‎integration-test/integration/chat/anthropic_test.clj‎

Lines changed: 308 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,308 @@
1+
(ns integration.chat.anthropic-test
2+
(:require
3+
[clojure.test :refer [deftest is testing]]
4+
[integration.eca :as eca]
5+
[integration.fixture :as fixture]
6+
[integration.helper :refer [match-content] :as h]
7+
[llm-mock.mocks :as llm.mocks]
8+
[matcher-combinators.matchers :as m]
9+
[matcher-combinators.test :refer [match?]]))
10+
11+
(eca/clean-after-test)
12+
13+
(deftest simple-text
14+
(eca/start-process!)
15+
16+
(eca/request! (fixture/initialize-request))
17+
(eca/notify! (fixture/initialized-notification))
18+
(let [chat-id* (atom nil)]
19+
(testing "We send a simple hello message"
20+
(llm.mocks/set-case! :simple-text-0)
21+
(let [req-id 0
22+
resp (eca/request! (fixture/chat-prompt-request
23+
{:request-id req-id
24+
:model "claude-sonnet-4-20250514"
25+
:message "Tell me a joke!"}))
26+
chat-id (reset! chat-id* (:chatId resp))]
27+
28+
(is (match?
29+
{:chatId (m/pred string?)
30+
:model "claude-sonnet-4-20250514"
31+
:status "success"}
32+
resp))
33+
34+
(match-content chat-id req-id "user" {:type "text" :text "Tell me a joke!\n"})
35+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
36+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
37+
(match-content chat-id req-id "assistant" {:type "text" :text "Knock"})
38+
(match-content chat-id req-id "assistant" {:type "text" :text " knock!"})
39+
(match-content chat-id req-id "system" {:type "usage"
40+
:messageInputTokens 10
41+
:messageOutputTokens 20
42+
:sessionTokens 30
43+
:messageCost (m/pred string?)
44+
:sessionCost (m/pred string?)})
45+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
46+
(is (match?
47+
{:messages [{:role "user" :content [{:type "text" :text "Tell me a joke!"}]}]
48+
:system (m/pred vector?)}
49+
llm.mocks/*last-req-body*))))
50+
51+
(testing "We reply"
52+
(llm.mocks/set-case! :simple-text-1)
53+
(let [req-id 1
54+
resp (eca/request! (fixture/chat-prompt-request
55+
{:chat-id @chat-id*
56+
:request-id req-id
57+
:model "claude-sonnet-4-20250514"
58+
:message "Who's there?"}))
59+
chat-id @chat-id*]
60+
61+
(is (match?
62+
{:chatId (m/pred string?)
63+
:model "claude-sonnet-4-20250514"
64+
:status "success"}
65+
resp))
66+
67+
(match-content chat-id req-id "user" {:type "text" :text "Who's there?\n"})
68+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
69+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
70+
(match-content chat-id req-id "assistant" {:type "text" :text "Foo"})
71+
(match-content chat-id req-id "system" {:type "usage"
72+
:messageInputTokens 10
73+
:messageOutputTokens 5
74+
:sessionTokens 45
75+
:messageCost (m/pred string?)
76+
:sessionCost (m/pred string?)})
77+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
78+
(is (match?
79+
{:messages [{:role "user" :content [{:type "text" :text "Tell me a joke!"}]}
80+
{:role "assistant" :content [{:type "text" :text "Knock knock!"}]}
81+
{:role "user" :content [{:type "text" :text "Who's there?"}]}]}
82+
llm.mocks/*last-req-body*))))
83+
84+
(testing "model reply again keeping context"
85+
(llm.mocks/set-case! :simple-text-2)
86+
(let [req-id 2
87+
resp (eca/request! (fixture/chat-prompt-request
88+
{:chat-id @chat-id*
89+
:request-id req-id
90+
:model "claude-sonnet-4-20250514"
91+
:message "What foo?"}))
92+
chat-id @chat-id*]
93+
94+
(is (match?
95+
{:chatId (m/pred string?)
96+
:model "claude-sonnet-4-20250514"
97+
:status "success"}
98+
resp))
99+
100+
(match-content chat-id req-id "user" {:type "text" :text "What foo?\n"})
101+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
102+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
103+
(match-content chat-id req-id "assistant" {:type "text" :text "Foo"})
104+
(match-content chat-id req-id "assistant" {:type "text" :text " bar!"})
105+
(match-content chat-id req-id "assistant" {:type "text" :text "\n\n"})
106+
(match-content chat-id req-id "assistant" {:type "text" :text "Ha!"})
107+
(match-content chat-id req-id "system" {:type "usage"
108+
:messageInputTokens 5
109+
:messageOutputTokens 15
110+
:sessionTokens 65
111+
:messageCost (m/pred string?)
112+
:sessionCost (m/pred string?)})
113+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
114+
(is (match?
115+
{:messages [{:role "user" :content [{:type "text" :text "Tell me a joke!"}]}
116+
{:role "assistant" :content [{:type "text" :text "Knock knock!"}]}
117+
{:role "user" :content [{:type "text" :text "Who's there?"}]}
118+
{:role "assistant" :content [{:type "text" :text "Foo"}]}
119+
{:role "user" :content [{:type "text" :text "What foo?"}]}]}
120+
llm.mocks/*last-req-body*))))))
121+
122+
(deftest reasoning-text
123+
(eca/start-process!)
124+
125+
(eca/request! (fixture/initialize-request))
126+
(eca/notify! (fixture/initialized-notification))
127+
(let [chat-id* (atom nil)]
128+
(testing "We send a hello message"
129+
(llm.mocks/set-case! :reasoning-0)
130+
(let [req-id 0
131+
resp (eca/request! (fixture/chat-prompt-request
132+
{:request-id req-id
133+
:model "claude-sonnet-4-20250514"
134+
:message "hello!"}))
135+
chat-id (reset! chat-id* (:chatId resp))]
136+
137+
(is (match?
138+
{:chatId (m/pred string?)
139+
:model "claude-sonnet-4-20250514"
140+
:status "success"}
141+
resp))
142+
143+
(match-content chat-id req-id "user" {:type "text" :text "hello!\n"})
144+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
145+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
146+
(match-content chat-id req-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
147+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text "I should say"})
148+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text " hello"})
149+
(match-content chat-id req-id "assistant" {:type "reasonFinished" :id (m/pred string?)})
150+
(match-content chat-id req-id "assistant" {:type "text" :text "hello"})
151+
(match-content chat-id req-id "assistant" {:type "text" :text " there!"})
152+
(match-content chat-id req-id "system" {:type "usage"
153+
:messageInputTokens 5
154+
:messageOutputTokens 30
155+
:sessionTokens 35
156+
:messageCost (m/pred string?)
157+
:sessionCost (m/pred string?)})
158+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
159+
(is (match?
160+
{:messages [{:role "user" :content [{:type "text" :text "hello!"}]}]
161+
:system (m/pred vector?)}
162+
llm.mocks/*last-req-body*))))
163+
164+
(testing "We reply"
165+
(llm.mocks/set-case! :reasoning-1)
166+
(let [req-id 1
167+
resp (eca/request! (fixture/chat-prompt-request
168+
{:request-id req-id
169+
:chat-id @chat-id*
170+
:model "claude-sonnet-4-20250514"
171+
:message "how are you?"}))
172+
chat-id @chat-id*]
173+
174+
(is (match?
175+
{:chatId (m/pred string?)
176+
:model "claude-sonnet-4-20250514"
177+
:status "success"}
178+
resp))
179+
180+
(match-content chat-id req-id "user" {:type "text" :text "how are you?\n"})
181+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
182+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
183+
(match-content chat-id req-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
184+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text "I should say"})
185+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text " fine"})
186+
(match-content chat-id req-id "assistant" {:type "reasonFinished" :id (m/pred string?)})
187+
(match-content chat-id req-id "assistant" {:type "text" :text "I'm "})
188+
(match-content chat-id req-id "assistant" {:type "text" :text " fine"})
189+
(match-content chat-id req-id "system" {:type "usage"
190+
:messageInputTokens 10
191+
:messageOutputTokens 20
192+
:sessionTokens 65
193+
:messageCost (m/pred string?)
194+
:sessionCost (m/pred string?)})
195+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
196+
(is (match?
197+
{:messages [{:role "user" :content [{:type "text" :text "hello!"}]}
198+
{:role "assistant"
199+
:content [{:type "thinking"
200+
:signature "enc-123"
201+
:thinking "I should say hello"}]}
202+
{:role "assistant" :content [{:type "text" :text "hello there!"}]}
203+
{:role "user" :content [{:type "text" :text "how are you?"}]}]
204+
:system (m/pred vector?)}
205+
llm.mocks/*last-req-body*))))))
206+
207+
(deftest tool-calling
208+
(eca/start-process!)
209+
210+
(eca/request! (fixture/initialize-request))
211+
(eca/notify! (fixture/initialized-notification))
212+
(let [chat-id* (atom nil)]
213+
(testing "We ask what files LLM see"
214+
(llm.mocks/set-case! :tool-calling-0)
215+
(let [req-id 0
216+
resp (eca/request! (fixture/chat-prompt-request
217+
{:request-id req-id
218+
:model "claude-sonnet-4-20250514"
219+
:message "What files you see?"}))
220+
chat-id (reset! chat-id* (:chatId resp))]
221+
222+
(is (match?
223+
{:chatId (m/pred string?)
224+
:model "claude-sonnet-4-20250514"
225+
:status "success"}
226+
resp))
227+
228+
(match-content chat-id req-id "user" {:type "text" :text "What files you see?\n"})
229+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
230+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
231+
(match-content chat-id req-id "assistant" {:type "reasonStarted" :id (m/pred string?)})
232+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text "I should call tool"})
233+
(match-content chat-id req-id "assistant" {:type "reasonText" :id (m/pred string?) :text " eca_directory_tree"})
234+
(match-content chat-id req-id "assistant" {:type "reasonFinished" :id (m/pred string?)})
235+
(match-content chat-id req-id "assistant" {:type "text" :text "I will list files"})
236+
(match-content chat-id req-id "assistant" {:type "toolCallPrepare"
237+
:origin "native"
238+
:id "tool-1"
239+
:name "eca_directory_tree"
240+
:argumentsText ""
241+
:manualApproval false
242+
:summary "Listing file tree"})
243+
(match-content chat-id req-id "assistant" {:type "toolCallPrepare"
244+
:origin "native"
245+
:id "tool-1"
246+
:name "eca_directory_tree"
247+
:argumentsText "{\"pat"
248+
:manualApproval false
249+
:summary "Listing file tree"})
250+
(match-content chat-id req-id "assistant" {:type "toolCallPrepare"
251+
:origin "native"
252+
:id "tool-1"
253+
:name "eca_directory_tree"
254+
:argumentsText (str "{\"path\":\"" (h/project-path->canon-path "resources") "\"}")
255+
:manualApproval false
256+
:summary "Listing file tree"})
257+
(match-content chat-id req-id "system" {:type "usage"
258+
:messageInputTokens 5
259+
:messageOutputTokens 30
260+
:sessionTokens 35
261+
:messageCost (m/pred string?)
262+
:sessionCost (m/pred string?)})
263+
(match-content chat-id req-id "assistant" {:type "toolCallRun"
264+
:origin "native"
265+
:id "tool-1"
266+
:name "eca_directory_tree"
267+
:arguments {:path (h/project-path->canon-path "resources")}
268+
:manualApproval false
269+
:summary "Listing file tree"})
270+
(match-content chat-id req-id "assistant" {:type "toolCalled"
271+
:origin "native"
272+
:id "tool-1"
273+
:name "eca_directory_tree"
274+
:arguments {:path (h/project-path->canon-path "resources")}
275+
:summary "Listing file tree"
276+
:error false
277+
:outputs [{:type "text" :text (str "[FILE] " (h/project-path->canon-path "resources/file2.md\n")
278+
"[FILE] " (h/project-path->canon-path "resources/file1.md\n"))}]})
279+
(match-content chat-id req-id "assistant" {:type "text" :text "The files I see:\n"})
280+
(match-content chat-id req-id "assistant" {:type "text" :text "file1\nfile2\n"})
281+
(match-content chat-id req-id "system" {:type "usage"
282+
:messageInputTokens 5
283+
:messageOutputTokens 30
284+
:sessionTokens 70
285+
:messageCost (m/pred string?)
286+
:sessionCost (m/pred string?)})
287+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
288+
(is (match?
289+
{:messages [{:role "user" :content [{:type "text" :text "What files you see?"}]}
290+
{:role "assistant"
291+
:content [{:type "thinking"
292+
:signature "enc-123"
293+
:thinking "I should call tool eca_directory_tree"}]}
294+
{:role "assistant" :content [{:type "text" :text "I will list files"}]}
295+
{:role "assistant"
296+
:content [{:type "tool_use"
297+
:id "tool-1"
298+
:name "eca_directory_tree"
299+
:input {:path (h/project-path->canon-path "resources")}}]}
300+
{:role "user"
301+
:content [{:type "tool_result"
302+
:tool_use_id "tool-1"
303+
:content (str "[FILE] " (h/project-path->canon-path "resources/file2.md\n")
304+
"[FILE] " (h/project-path->canon-path "resources/file1.md\n\n"))}]}]
305+
:tools (m/embeds
306+
[{:name "eca_directory_tree"}])
307+
:system (m/pred vector?)}
308+
llm.mocks/*last-req-body*))))))

0 commit comments

Comments
 (0)