Skip to content

Commit 26d67bf

Browse files
ericdalloericdallo
committed
tests: add reasoning openai integration test
1 parent e1543d1 commit 26d67bf

File tree

2 files changed

+162
-17
lines changed

2 files changed

+162
-17
lines changed

‎integration-test/integration/chat_openai_test.clj‎

Lines changed: 91 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -28,13 +28,13 @@
2828
(let [req-id 0
2929
resp (eca/request! (fixture/chat-prompt-request
3030
{:request-id req-id
31-
:model "gpt-5"
31+
:model "gpt-4.1"
3232
:message "Tell me a joke!"}))
3333
chat-id (reset! chat-id* (:chatId resp))]
3434

3535
(is (match?
3636
{:chatId (m/pred string?)
37-
:model "gpt-5"
37+
:model "gpt-4.1"
3838
:status "success"}
3939
resp))
4040

@@ -61,13 +61,13 @@
6161
resp (eca/request! (fixture/chat-prompt-request
6262
{:chat-id @chat-id*
6363
:request-id req-id
64-
:model "gpt-5"
64+
:model "gpt-4.1"
6565
:message "Who's there?"}))
6666
chat-id @chat-id*]
6767

6868
(is (match?
6969
{:chatId (m/pred string?)
70-
:model "gpt-5"
70+
:model "gpt-4.1"
7171
:status "success"}
7272
resp))
7373

@@ -94,13 +94,13 @@
9494
resp (eca/request! (fixture/chat-prompt-request
9595
{:chat-id @chat-id*
9696
:request-id req-id
97-
:model "gpt-5"
97+
:model "gpt-4.1"
9898
:message "What foo?"}))
9999
chat-id @chat-id*]
100100

101101
(is (match?
102102
{:chatId (m/pred string?)
103-
:model "gpt-5"
103+
:model "gpt-4.1"
104104
:status "success"}
105105
resp))
106106

@@ -125,3 +125,88 @@
125125
{:role "assistant" :content [{:type "output_text" :text "Foo"}]}
126126
{:role "user" :content [{:type "input_text" :text "What foo?"}]}]}
127127
llm.mocks/*last-req-body*))))))
128+
129+
(deftest reasoning-text
130+
(eca/start-process!)
131+
132+
(eca/request! (fixture/initialize-request))
133+
(eca/notify! (fixture/initialized-notification))
134+
(let [chat-id* (atom nil)]
135+
(testing "We send a hello message"
136+
(llm.mocks/set-case! :reasoning-0)
137+
(let [req-id 0
138+
resp (eca/request! (fixture/chat-prompt-request
139+
{:request-id req-id
140+
:model "gpt-5"
141+
:message "hello!"}))
142+
chat-id (reset! chat-id* (:chatId resp))]
143+
144+
(is (match?
145+
{:chatId (m/pred string?)
146+
:model "gpt-5"
147+
:status "success"}
148+
resp))
149+
150+
(match-content chat-id req-id "user" {:type "text" :text "hello!\n"})
151+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
152+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
153+
(match-content chat-id req-id "assistant" {:type "reasonStarted" :id "123"})
154+
(match-content chat-id req-id "assistant" {:type "reasonText" :id "123" :text "I should say"})
155+
(match-content chat-id req-id "assistant" {:type "reasonText" :id "123" :text " hello"})
156+
(match-content chat-id req-id "assistant" {:type "reasonFinished" :id "123"})
157+
(match-content chat-id req-id "assistant" {:type "text" :text "hello"})
158+
(match-content chat-id req-id "assistant" {:type "text" :text " there!"})
159+
(match-content chat-id req-id "system" {:type "usage"
160+
:messageInputTokens 5
161+
:messageOutputTokens 30
162+
:sessionTokens 35
163+
:messageCost (m/pred string?)
164+
:sessionCost (m/pred string?)})
165+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
166+
(is (match?
167+
{:input [{:role "user" :content [{:type "input_text" :text "hello!"}]}]
168+
:instructions (m/pred string?)}
169+
llm.mocks/*last-req-body*))))
170+
171+
(testing "We reply"
172+
(llm.mocks/set-case! :reasoning-1)
173+
(let [req-id 1
174+
resp (eca/request! (fixture/chat-prompt-request
175+
{:request-id req-id
176+
:chat-id @chat-id*
177+
:model "gpt-5"
178+
:message "how are you?"}))
179+
chat-id @chat-id*]
180+
181+
(is (match?
182+
{:chatId (m/pred string?)
183+
:model "gpt-5"
184+
:status "success"}
185+
resp))
186+
187+
(match-content chat-id req-id "user" {:type "text" :text "how are you?\n"})
188+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Waiting model"})
189+
(match-content chat-id req-id "system" {:type "progress" :state "running" :text "Generating"})
190+
(match-content chat-id req-id "assistant" {:type "reasonStarted" :id "234"})
191+
(match-content chat-id req-id "assistant" {:type "reasonText" :id "234" :text "I should say"})
192+
(match-content chat-id req-id "assistant" {:type "reasonText" :id "234" :text " fine"})
193+
(match-content chat-id req-id "assistant" {:type "reasonFinished" :id "234"})
194+
(match-content chat-id req-id "assistant" {:type "text" :text "I'm "})
195+
(match-content chat-id req-id "assistant" {:type "text" :text " fine"})
196+
(match-content chat-id req-id "system" {:type "usage"
197+
:messageInputTokens 10
198+
:messageOutputTokens 20
199+
:sessionTokens 65
200+
:messageCost (m/pred string?)
201+
:sessionCost (m/pred string?)})
202+
(match-content chat-id req-id "system" {:type "progress" :state "finished"})
203+
(is (match?
204+
{:input [{:role "user" :content [{:type "input_text" :text "hello!"}]}
205+
{:type "reasoning"
206+
:id "123"
207+
:summary [{:type "summary_text" :text "I should say hello"}]
208+
:encrypted_content "enc-123"}
209+
{:role "assistant" :content [{:type "output_text" :text "hello there!"}]}
210+
{:role "user" :content [{:type "input_text" :text "how are you?"}]}]
211+
:instructions (m/pred string?)}
212+
llm.mocks/*last-req-body*))))))

‎integration-test/llm_mock/openai.clj‎

Lines changed: 71 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -52,19 +52,79 @@
5252
:status "completed"}})
5353
(hk/close ch))
5454

55+
(defn ^:private reasoning-0 [ch]
56+
(sse-send! ch "response.output_item.added"
57+
{:type "response.output_item.added"
58+
:item {:type "reasoning" :id "123"}})
59+
(sse-send! ch "response.reasoning_summary_text.delta"
60+
{:type "response.reasoning_summary_text.delta"
61+
:item_id "123"
62+
:delta "I should say"})
63+
(sse-send! ch "response.reasoning_summary_text.delta"
64+
{:type "response.reasoning_summary_text.delta"
65+
:item_id "123"
66+
:delta " hello"})
67+
(sse-send! ch "response.output_item.done"
68+
{:type "response.output_item.done"
69+
:item {:type "reasoning"
70+
:id "123"
71+
:encrypted_content "enc-123"}})
72+
(sse-send! ch "response.output_text.delta"
73+
{:type "response.output_text.delta" :delta "hello"})
74+
(sse-send! ch "response.output_text.delta"
75+
{:type "response.output_text.delta" :delta " there!"})
76+
(sse-send! ch "response.completed"
77+
{:type "response.completed"
78+
:response {:output []
79+
:usage {:input_tokens 5
80+
:output_tokens 30}
81+
:status "completed"}})
82+
(hk/close ch))
83+
84+
(defn ^:private reasoning-1 [ch]
85+
(sse-send! ch "response.output_item.added"
86+
{:type "response.output_item.added"
87+
:item {:type "reasoning" :id "234"}})
88+
(sse-send! ch "response.reasoning_summary_text.delta"
89+
{:type "response.reasoning_summary_text.delta"
90+
:item_id "234"
91+
:delta "I should say"})
92+
(sse-send! ch "response.reasoning_summary_text.delta"
93+
{:type "response.reasoning_summary_text.delta"
94+
:item_id "234"
95+
:delta " fine"})
96+
(sse-send! ch "response.output_item.done"
97+
{:type "response.output_item.done"
98+
:item {:type "reasoning"
99+
:id "234"
100+
:encrypted_content "enc-234"}})
101+
(sse-send! ch "response.output_text.delta"
102+
{:type "response.output_text.delta" :delta "I'm "})
103+
(sse-send! ch "response.output_text.delta"
104+
{:type "response.output_text.delta" :delta " fine"})
105+
(sse-send! ch "response.completed"
106+
{:type "response.completed"
107+
:response {:output []
108+
:usage {:input_tokens 10
109+
:output_tokens 20}
110+
:status "completed"}})
111+
(hk/close ch))
112+
55113
(defn handle-openai-responses [req]
56114
(llm.mocks/set-last-req-body! (some-> (slurp (:body req))
57115
(json/parse-string true)))
58116
(hk/as-channel
59-
req
60-
{:on-open (fn [ch]
117+
req
118+
{:on-open (fn [ch]
61119
;; initial SSE handshake
62-
(hk/send! ch {:status 200
63-
:headers {"Content-Type" "text/event-stream; charset=utf-8"
64-
"Cache-Control" "no-cache"
65-
"Connection" "keep-alive"}}
66-
false)
67-
(case llm.mocks/*case*
68-
:simple-text-0 (simple-text-0 ch)
69-
:simple-text-1 (simple-text-1 ch)
70-
:simple-text-2 (simple-text-2 ch)))}))
120+
(hk/send! ch {:status 200
121+
:headers {"Content-Type" "text/event-stream; charset=utf-8"
122+
"Cache-Control" "no-cache"
123+
"Connection" "keep-alive"}}
124+
false)
125+
(case llm.mocks/*case*
126+
:simple-text-0 (simple-text-0 ch)
127+
:simple-text-1 (simple-text-1 ch)
128+
:simple-text-2 (simple-text-2 ch)
129+
:reasoning-0 (reasoning-0 ch)
130+
:reasoning-1 (reasoning-1 ch)))}))

0 commit comments

Comments
 (0)