Skip to content

Commit 8b82bf1

Browse files
ericdalloericdallo
committed
tests: Add tool-calling openai integration tests
1 parent b1b55e5 commit 8b82bf1

File tree

7 files changed

+206
-24
lines changed

7 files changed

+206
-24
lines changed

‎docs/protocol.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -657,7 +657,7 @@ interface ToolCalledContent {
657657
/**
658658
* The content of this output
659659
*/
660-
content: string;
660+
text: string;
661661
}];
662662

663663
/**

‎integration-test/entrypoint.clj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@
55
[llm-mock.server :as llm-mock.server]))
66

77
(def namespaces
8-
'[integration.initialize-test
8+
'[;integration.initialize-test
99
integration.chat.openai-test])
1010

1111
(defn timeout [timeout-ms callback]

‎integration-test/integration/chat/openai_test.clj‎

Lines changed: 102 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -3,20 +3,13 @@
33
[clojure.test :refer [deftest is testing]]
44
[integration.eca :as eca]
55
[integration.fixture :as fixture]
6+
[integration.helper :refer [match-content] :as h]
67
[llm-mock.mocks :as llm.mocks]
78
[matcher-combinators.matchers :as m]
89
[matcher-combinators.test :refer [match?]]))
910

1011
(eca/clean-after-test)
1112

12-
(defn match-content [chat-id request-id role content]
13-
(is (match?
14-
{:chatId chat-id
15-
:requestId request-id
16-
:role role
17-
:content content}
18-
(eca/client-awaits-server-notification :chat/contentReceived))))
19-
2013
(deftest simple-text
2114
(eca/start-process!)
2215

@@ -210,3 +203,104 @@
210203
{:role "user" :content [{:type "input_text" :text "how are you?"}]}]
211204
:instructions (m/pred string?)}
212205
llm.mocks/*last-req-body*))))))
206+
207+
(deftest tool-calling
208+
(eca/start-process!)
209+
210+
(eca/request! (fixture/initialize-request))
211+
(eca/notify! (fixture/initialized-notification))
212+
(let [chat-id* (atom nil)]
213+
(testing "We ask what files LLM see"
214+
(llm.mocks/set-case! :tool-calling-0)
215+
(let [req-id 0
216+
resp (eca/request! (fixture/chat-prompt-request
217+
{:request-id req-id
218+
:model "gpt-5"
219+
:message "What files you see?"}))
220+
chat-id (reset! chat-id* (:chatId resp))]
221+
222+
(is (match?
223+
{:chatId (m/pred string?)
224+
:model "gpt-5"
225+
:status "success"}
226+
resp))
227+
228+
(match-content chat-id req-id "user" {:type "text" :text "What files you see?\n"})
229+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
230+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
231+
(match-content chat-id req-id "assistant" {:type "reasonStarted" :id "123"})
232+
(match-content chat-id req-id "assistant" {:type "reasonText" :id "123" :text "I should call tool"})
233+
(match-content chat-id req-id "assistant" {:type "reasonText" :id "123" :text " eca_directory_tree"})
234+
(match-content chat-id req-id "assistant" {:type "reasonFinished" :id "123"})
235+
(match-content chat-id req-id "assistant" {:type "text" :text "I will list files"})
236+
(match-content chat-id req-id "assistant" {:type "toolCallPrepare"
237+
:origin "native"
238+
:id "tool-1"
239+
:name "eca_directory_tree"
240+
:argumentsText ""
241+
:manualApproval false
242+
:summary "Listing file tree"})
243+
(match-content chat-id req-id "assistant" {:type "toolCallPrepare"
244+
:origin "native"
245+
:id "tool-1"
246+
:name "eca_directory_tree"
247+
:argumentsText "{\"pat"
248+
:manualApproval false
249+
:summary "Listing file tree"})
250+
(match-content chat-id req-id "assistant" {:type "toolCallPrepare"
251+
:origin "native"
252+
:id "tool-1"
253+
:name "eca_directory_tree"
254+
:argumentsText (str "{\"path\":\"" (h/project-path->canon-path "resources") "\"}")
255+
:manualApproval false
256+
:summary "Listing file tree"})
257+
(match-content chat-id req-id "system" {:type "usage"
258+
:messageInputTokens 5
259+
:messageOutputTokens 30
260+
:sessionTokens 35
261+
:messageCost (m/pred string?)
262+
:sessionCost (m/pred string?)})
263+
(match-content chat-id req-id "assistant" {:type "toolCallRun"
264+
:origin "native"
265+
:id "tool-1"
266+
:name "eca_directory_tree"
267+
:arguments {:path (h/project-path->canon-path "resources")}
268+
:manualApproval false
269+
:summary "Listing file tree"})
270+
(match-content chat-id req-id "assistant" {:type "toolCalled"
271+
:origin "native"
272+
:id "tool-1"
273+
:name "eca_directory_tree"
274+
:arguments {:path (h/project-path->canon-path "resources")}
275+
:summary "Listing file tree"
276+
:error false
277+
:outputs [{:type "text" :text (str "[FILE] " (h/project-path->canon-path "resources/file2.md\n")
278+
"[FILE] " (h/project-path->canon-path "resources/file1.md\n"))}]})
279+
(match-content chat-id req-id "assistant" {:type "text" :text "The files I see:\n"})
280+
(match-content chat-id req-id "assistant" {:type "text" :text "file1\nfile2\n"})
281+
(match-content chat-id req-id "system" {:type "usage"
282+
:messageInputTokens 5
283+
:messageOutputTokens 30
284+
:sessionTokens 70
285+
:messageCost (m/pred string?)
286+
:sessionCost (m/pred string?)})
287+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
288+
(is (match?
289+
{:input [{:role "user" :content [{:type "input_text" :text "What files you see?"}]}
290+
{:type "reasoning"
291+
:id "123"
292+
:summary [{:type "summary_text" :text "I should call tool eca_directory_tree"}]
293+
:encrypted_content "enc-123"}
294+
{:role "assistant" :content [{:type "output_text" :text "I will list files"}]}
295+
{:type "function_call"
296+
:name "eca_directory_tree"
297+
:call_id "tool-1"
298+
:arguments (str "{\"path\":\"" (h/project-path->canon-path "resources") "\"}")}
299+
{:type "function_call_output"
300+
:call_id "tool-1"
301+
:output (str "[FILE] " (h/project-path->canon-path "resources/file2.md\n")
302+
"[FILE] " (h/project-path->canon-path "resources/file1.md\n\n"))}]
303+
:tools (m/embeds
304+
[{:name "eca_directory_tree"}])
305+
:instructions (m/pred string?)}
306+
llm.mocks/*last-req-body*))))))

‎integration-test/integration/helper.clj‎

Lines changed: 17 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,10 @@
22
(:require
33
[babashka.fs :as fs]
44
[clojure.java.io :as io]
5-
[clojure.string :as string]))
5+
[clojure.string :as string]
6+
[clojure.test :refer [is]]
7+
[integration.eca :as eca]
8+
[matcher-combinators.test :refer [match?]]))
69

710
(def windows?
811
"Whether is running on MS-Windows."
@@ -18,6 +21,11 @@
1821
fs/canonicalize
1922
str))
2023

24+
(defn project-path->canon-path
25+
"Returns the canonical name of the root project's SUB-PATH."
26+
[sub-path]
27+
(.getCanonicalPath (io/file default-root-project-path sub-path)))
28+
2129
(defn escape-uri
2230
"Escapes enough URI characters for testing purposes and returns it.
2331
@@ -34,3 +42,11 @@
3442
(if *escape-uris?*
3543
(escape-uri uri)
3644
uri)))
45+
46+
(defn match-content [chat-id request-id role content]
47+
(is (match?
48+
{:chatId chat-id
49+
:requestId request-id
50+
:role role
51+
:content content}
52+
(eca/client-awaits-server-notification :chat/contentReceived))))

‎integration-test/llm_mock/openai.clj‎

Lines changed: 83 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
(ns llm-mock.openai
22
(:require
33
[cheshire.core :as json]
4+
[integration.helper :as h]
45
[llm-mock.mocks :as llm.mocks]
56
[org.httpkit.server :as hk]))
67

@@ -110,21 +111,90 @@
110111
:status "completed"}})
111112
(hk/close ch))
112113

114+
(defn ^:private tool-calling-0 [ch]
115+
(let [body llm.mocks/*last-req-body*
116+
second-stage? (some #(= "function_call_output" (:type %)) (:input body))]
117+
(if-not second-stage?
118+
(let [args-json (json/generate-string {:path (h/project-path->canon-path "resources")})]
119+
;; Reasoning prelude
120+
(sse-send! ch "response.output_item.added"
121+
{:type "response.output_item.added"
122+
:item {:type "reasoning" :id "123"}})
123+
(sse-send! ch "response.reasoning_summary_text.delta"
124+
{:type "response.reasoning_summary_text.delta"
125+
:item_id "123"
126+
:delta "I should call tool"})
127+
(sse-send! ch "response.reasoning_summary_text.delta"
128+
{:type "response.reasoning_summary_text.delta"
129+
:item_id "123"
130+
:delta " eca_directory_tree"})
131+
(sse-send! ch "response.output_item.done"
132+
{:type "response.output_item.done"
133+
:item {:type "reasoning"
134+
:id "123"
135+
:encrypted_content "enc-123"}})
136+
;; Short text before tool call
137+
(sse-send! ch "response.output_text.delta"
138+
{:type "response.output_text.delta" :delta "I will list files"})
139+
;; Function call announced
140+
(sse-send! ch "response.output_item.added"
141+
{:type "response.output_item.added"
142+
:item {:type "function_call"
143+
:id "item-1"
144+
:call_id "tool-1"
145+
:name "eca_directory_tree"
146+
:arguments ""}})
147+
;; Stream arguments in two chunks
148+
(sse-send! ch "response.function_call_arguments.delta"
149+
{:type "response.function_call_arguments.delta"
150+
:item_id "item-1"
151+
:delta "{\"pat"})
152+
(sse-send! ch "response.function_call_arguments.delta"
153+
{:type "response.function_call_arguments.delta"
154+
:item_id "item-1"
155+
:delta (str "h\":\"" (h/project-path->canon-path "resources") "\"}")})
156+
;; Complete with the function call present so the client triggers tools
157+
(sse-send! ch "response.completed"
158+
{:type "response.completed"
159+
:response {:output [{:type "function_call"
160+
:id "item-1"
161+
:call_id "tool-1"
162+
:name "eca_directory_tree"
163+
:arguments args-json}]
164+
:usage {:input_tokens 5
165+
:output_tokens 30}
166+
:status "completed"}})
167+
(hk/close ch))
168+
;; Second stage: after tool outputs are supplied back to the model
169+
(do
170+
(sse-send! ch "response.output_text.delta"
171+
{:type "response.output_text.delta" :delta "The files I see:\n"})
172+
(sse-send! ch "response.output_text.delta"
173+
{:type "response.output_text.delta" :delta "file1\nfile2\n"})
174+
(sse-send! ch "response.completed"
175+
{:type "response.completed"
176+
:response {:output []
177+
:usage {:input_tokens 5
178+
:output_tokens 30}
179+
:status "completed"}})
180+
(hk/close ch)))))
181+
113182
(defn handle-openai-responses [req]
114183
(llm.mocks/set-last-req-body! (some-> (slurp (:body req))
115184
(json/parse-string true)))
116185
(hk/as-channel
117-
req
118-
{:on-open (fn [ch]
186+
req
187+
{:on-open (fn [ch]
119188
;; initial SSE handshake
120-
(hk/send! ch {:status 200
121-
:headers {"Content-Type" "text/event-stream; charset=utf-8"
122-
"Cache-Control" "no-cache"
123-
"Connection" "keep-alive"}}
124-
false)
125-
(case llm.mocks/*case*
126-
:simple-text-0 (simple-text-0 ch)
127-
:simple-text-1 (simple-text-1 ch)
128-
:simple-text-2 (simple-text-2 ch)
129-
:reasoning-0 (reasoning-0 ch)
130-
:reasoning-1 (reasoning-1 ch)))}))
189+
(hk/send! ch {:status 200
190+
:headers {"Content-Type" "text/event-stream; charset=utf-8"
191+
"Cache-Control" "no-cache"
192+
"Connection" "keep-alive"}}
193+
false)
194+
(case llm.mocks/*case*
195+
:simple-text-0 (simple-text-0 ch)
196+
:simple-text-1 (simple-text-1 ch)
197+
:simple-text-2 (simple-text-2 ch)
198+
:reasoning-0 (reasoning-0 ch)
199+
:reasoning-1 (reasoning-1 ch)
200+
:tool-calling-0 (tool-calling-0 ch)))}))

‎integration-test/sample-test/resources/file1.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Something here

‎integration-test/sample-test/resources/file2.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
Other thing here

0 commit comments

Comments
 (0)