Skip to content

Commit 3268452

Browse files
ericdalloericdallo
committed
Support reason/thinking
1 parent eea9d48 commit 3268452

File tree

8 files changed

+108
-29
lines changed

8 files changed

+108
-29
lines changed

‎CHANGELOG.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@
33
## Unreleased
44

55
- Increase anthropic models maxTokens to 8196
6+
- Support thinking/reasoning on models that support it.
67

78
## 0.9.0
89

‎build.clj‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -45,6 +45,7 @@
4545
OPTS can be a map of
4646
:jvm-opts A vector of options ot pass to the JVM."
4747
[opts]
48+
(io/delete-file "eca")
4849
(println "Generating bin...")
4950
(let [jvm-opts (concat (:jvm-opts opts []) ["-server"])]
5051
((requiring-resolve 'deps-bin.impl.bin/build-bin)

‎docs/protocol.md‎

Lines changed: 47 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -377,6 +377,9 @@ type ChatContent =
377377
| ProgressContent
378378
| UsageContent
379379
| FileChangeContent
380+
| ReasonStartedContent
381+
| ReasonTextContent
382+
| ReasonFinishedContent
380383
| ToolCallPrepareContent
381384
| ToolCallRunContent
382385
| ToolCalledContent;
@@ -392,6 +395,50 @@ interface TextContent {
392395
text: string;
393396
}
394397

398+
/**
399+
* A reason started from the LLM
400+
*
401+
*/
402+
interface ReasonStartedContent {
403+
type: 'reasonStarted';
404+
405+
/**
406+
* The id of this reason
407+
*/
408+
id: string;
409+
}
410+
411+
/**
412+
* A reason text from the LLM
413+
*
414+
*/
415+
interface ReasonTextContent {
416+
type: 'reasonText';
417+
418+
/**
419+
* The id of a started reason
420+
*/
421+
id: string;
422+
423+
/**
424+
* The text content of the reasoning
425+
*/
426+
text: string;
427+
}
428+
429+
/**
430+
* A reason finished from the LLM
431+
*
432+
*/
433+
interface ReasonFinishedContent {
434+
type: 'reasonFinished';
435+
436+
/**
437+
* The id of this reason
438+
*/
439+
id: string;
440+
}
441+
395442
/**
396443
* URL content message from the LLM
397444
*/

‎src/eca/db.clj‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,11 +27,13 @@
2727
"claude-sonnet-4-0" {:tools true
2828
:web-search true
2929
:max-output-tokens 8196
30+
:reason-tokens 2048
3031
:input-token-cost (/ 3.0 one-million)
3132
:output-token-cost (/ 15.0 one-million)}
3233
"claude-opus-4-0" {:tools true
3334
:web-search true
3435
:max-output-tokens 8196
36+
:reason-tokens 2048
3537
:input-token-cost (/ 15.0 one-million)
3638
:output-token-cost (/ 75.0 one-million)}
3739
"claude-3-5-haiku-latest" {:tools true

‎src/eca/features/chat.clj‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -218,16 +218,20 @@
218218
:reason :user
219219
:id id})
220220
{:new-messages (get-in @db* [:chats chat-id :messages])}))))
221-
:on-reason (fn [{:keys [status]}]
221+
:on-reason (fn [{:keys [status id text]}]
222222
(assert-chat-not-stopped! chat-ctx)
223-
(let [msg (case status
224-
:started "Reasoning"
225-
:finished "Waiting model"
226-
nil)]
227-
(send-content! chat-ctx :system
228-
{:type :progress
229-
:state :running
230-
:text msg})))
223+
(case status
224+
:started (send-content! chat-ctx :assistant
225+
{:type :reasonStarted
226+
:id id})
227+
:thinking (send-content! chat-ctx :assistant
228+
{:type :reasonText
229+
:id id
230+
:text text})
231+
:finished (send-content! chat-ctx :assistant
232+
{:type :reasonFinished
233+
:id id})
234+
nil))
231235
:on-error (fn [{:keys [message exception]}]
232236
(send-content! chat-ctx :system
233237
{:type :text

‎src/eca/llm_api.clj‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -87,6 +87,7 @@
8787
(mapv tool->llm-tool tools))
8888
web-search (:web-search model-config)
8989
max-output-tokens (:max-output-tokens model-config)
90+
reason-tokens (:reason-tokens model-config)
9091
custom-providers (:customProviders config)
9192
custom-models (set (mapcat (fn [[k v]]
9293
(map #(str (name k) "/" %) (:models v)))
@@ -105,6 +106,7 @@
105106
:instructions instructions
106107
:user-prompt user-prompt
107108
:max-output-tokens max-output-tokens
109+
:reason-tokens reason-tokens
108110
:past-messages past-messages
109111
:tools tools
110112
:web-search web-search
@@ -120,6 +122,7 @@
120122
:instructions instructions
121123
:user-prompt user-prompt
122124
:max-output-tokens max-output-tokens
125+
:reason-tokens reason-tokens
123126
:past-messages past-messages
124127
:tools tools
125128
:web-search web-search
@@ -152,6 +155,7 @@
152155
:instructions instructions
153156
:user-prompt user-prompt
154157
:max-output-tokens max-output-tokens
158+
:reason-tokens reason-tokens
155159
:past-messages past-messages
156160
:web-search web-search
157161
:tools tools

‎src/eca/llm_providers/anthropic.clj‎

Lines changed: 32 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
[clojure.java.io :as io]
55
[eca.llm-util :as llm-util]
66
[eca.logger :as logger]
7-
[eca.shared :as shared]
7+
[eca.shared :as shared :refer [assoc-some]]
88
[hato.client :as http]))
99

1010
(set! *warn-on-reflection* true)
@@ -26,7 +26,8 @@
2626
:cache_control {:type "ephemeral"}})))
2727

2828
(defn ^:private base-request! [{:keys [rid body api-url api-key content-block* on-error on-response]}]
29-
(let [url (str api-url messages-path)]
29+
(let [url (str api-url messages-path)
30+
reason-id (str (random-uuid))]
3031
(llm-util/log-request logger-tag rid url body)
3132
(http/post
3233
url
@@ -46,7 +47,7 @@
4647
(with-open [rdr (io/reader body)]
4748
(doseq [[event data] (llm-util/event-data-seq rdr)]
4849
(llm-util/log-response logger-tag rid event data)
49-
(on-response event data content-block*))))
50+
(on-response event data content-block* reason-id))))
5051
(catch Exception e
5152
(on-error {:exception e}))))
5253
(fn [e]
@@ -77,31 +78,45 @@
7778

7879
(defn completion!
7980
[{:keys [model user-prompt temperature instructions max-output-tokens
80-
api-url api-key past-messages tools web-search]
81+
api-url api-key reason-tokens past-messages tools web-search]
8182
:or {temperature 1.0}}
82-
{:keys [on-message-received on-error on-prepare-tool-call on-tool-called]}]
83+
{:keys [on-message-received on-error on-reason on-prepare-tool-call on-tool-called]}]
8384
(let [messages (conj (past-messages->messages past-messages)
8485
{:role "user" :content [{:type :text
8586
:text user-prompt}]})
86-
body {:model model
87-
:messages (add-cache-to-last-message messages)
88-
:max_tokens max-output-tokens
89-
:temperature temperature
90-
;; TODO support :thinking
91-
:stream true
92-
:tools (->tools tools web-search)
93-
:system [{:type "text" :text instructions :cache_control {:type "ephemeral"}}]}
87+
body (assoc-some
88+
{:model model
89+
:messages (add-cache-to-last-message messages)
90+
:max_tokens max-output-tokens
91+
:temperature temperature
92+
:stream true
93+
:tools (->tools tools web-search)
94+
:system [{:type "text" :text instructions :cache_control {:type "ephemeral"}}]}
95+
:thinking (when (and reason-tokens (> reason-tokens 0))
96+
{:type "enabled"
97+
:budget_tokens reason-tokens}))
98+
9499
on-response-fn
95-
(fn handle-response [event data content-block*]
100+
(fn handle-response [event data content-block* reason-id]
96101
(case event
97102
"content_block_start" (case (-> data :content_block :type)
103+
"thinking" (do
104+
(on-reason {:status :started
105+
:id reason-id})
106+
(swap! content-block* assoc (:index data) (:content_block data)))
98107
"tool_use" (do
99108
(on-prepare-tool-call {:name (-> data :content_block :name)
100109
:id (-> data :content_block :id)
101110
:arguments-text ""})
102111
(swap! content-block* assoc (:index data) (:content_block data)))
103112

104113
nil)
114+
"content_block_stop" (when-let [content-block (get @content-block* (:index data))]
115+
(case (:type content-block)
116+
"thinking" (on-reason {:status :finished
117+
:id reason-id})
118+
nil)
119+
(swap! content-block* dissoc (:index data)))
105120
"content_block_delta" (case (-> data :delta :type)
106121
"text_delta" (on-message-received {:type :text
107122
:text (-> data :delta :text)})
@@ -117,6 +132,9 @@
117132
:title (-> data :delta :citation :title)
118133
:url (-> data :delta :citation :url)})
119134
nil)
135+
"thinking_delta" (on-reason {:status :thinking
136+
:id reason-id
137+
:text (-> data :delta :thinking)})
120138
(logger/warn "Unkown response delta type" (-> data :delta :type)))
121139
"message_delta" (case (-> data :delta :stop_reason)
122140
"tool_use" (doseq [content-block (vals @content-block*)]

‎src/eca/llm_providers/openai.clj‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,6 @@
44
[clojure.java.io :as io]
55
[eca.llm-util :as llm-util]
66
[eca.logger :as logger]
7-
[eca.shared :refer [assoc-some]]
87
[hato.client :as http]))
98

109
(set! *warn-on-reflection* true)
@@ -16,7 +15,8 @@
1615
(def base-url "https://api.openai.com")
1716

1817
(defn ^:private base-completion-request! [{:keys [rid body api-url api-key on-error on-response]}]
19-
(let [url (str api-url responses-path)]
18+
(let [url (str api-url responses-path)
19+
reason-id (str (random-uuid))]
2020
(llm-util/log-request logger-tag rid url body)
2121
(http/post
2222
url
@@ -35,7 +35,7 @@
3535
(with-open [rdr (io/reader body)]
3636
(doseq [[event data] (llm-util/event-data-seq rdr)]
3737
(llm-util/log-response logger-tag rid event data)
38-
(on-response event data))))
38+
(on-response event data reason-id))))
3939
(catch Exception e
4040
(on-error {:exception e}))))
4141
(fn [e]
@@ -73,7 +73,7 @@
7373
:max_completion_tokens max-output-tokens}
7474
mcp-call-by-item-id* (atom {})
7575
on-response-fn
76-
(fn handle-response [event data]
76+
(fn handle-response [event data reason-id]
7777
(case event
7878
;; text
7979
"response.output_text.delta"
@@ -101,7 +101,8 @@
101101
:on-error on-error
102102
:on-response handle-response})
103103
(swap! mcp-call-by-item-id* dissoc (-> data :item :id)))
104-
"reasoning" (on-reason {:status :finished})
104+
"reasoning" (on-reason {:status :finished
105+
:id reason-id})
105106
nil)
106107

107108
;; URL mentioned
@@ -116,7 +117,8 @@
116117
;; reasoning / tools
117118
"response.output_item.added"
118119
(case (-> data :item :type)
119-
"reasoning" (on-reason {:status :started})
120+
"reasoning" (on-reason {:status :started
121+
:id reason-id})
120122
"function_call" (let [call-id (-> data :item :call_id)
121123
item-id (-> data :item :id)
122124
name (-> data :item :name)]

0 commit comments

Comments
 (0)