Skip to content

Commit 772ac63

Browse files
ericdalloericdallo
committed
Support ollama
1 parent 6a02ddb commit 772ac63

File tree

4 files changed

+60
-43
lines changed

4 files changed

+60
-43
lines changed

‎integration-test/llm_mock/ollama.clj‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,10 @@
6363
(hk/close ch)))))
6464

6565
(defn ^:private chat-title-text-0 [ch]
66-
(sse-send! ch {:message {:thinking "Some Cool"}})
67-
(sse-send! ch {:message {:thinking " Title"}})
68-
(sse-send! ch {:done_reason "stop"})
69-
(hk/close ch))
66+
(hk/send! ch
67+
(json/generate-string
68+
{:message {:content "Some Cool Title"}})
69+
true))
7070

7171
(defn handle-ollama-chat [req]
7272
(let [body-str (slurp (:body req))

‎src/eca/llm_providers/ollama.clj‎

Lines changed: 45 additions & 31 deletions
Original file line numberDiff line numberDiff line change
@@ -51,30 +51,42 @@
5151
(logger/warn logger-tag "Error getting model:" (ex-message e))
5252
[])))
5353

54-
(defn ^:private base-chat-request! [{:keys [rid url body on-error on-response]}]
55-
(llm-util/log-request logger-tag rid url body)
54+
(defn ^:private base-chat-request! [{:keys [rid url body on-error on-stream]}]
5655
(let [reason-id (str (random-uuid))
57-
reasoning?* (atom false)]
58-
(http/post
59-
url
60-
{:body (json/generate-string body)
61-
:throw-exceptions? false
62-
:async? true
63-
:as :stream}
64-
(fn [{:keys [status body]}]
65-
(try
66-
(if (not= 200 status)
67-
(let [body-str (slurp body)]
68-
(logger/warn logger-tag (format "Unexpected response status: %s body: %s" status body-str))
69-
(on-error {:message (format "Ollama response status: %s body: %s" status body-str)}))
70-
(with-open [rdr (io/reader body)]
71-
(doseq [[event data] (llm-util/event-data-seq rdr)]
72-
(llm-util/log-response logger-tag rid event data)
73-
(on-response rid event data reasoning?* reason-id))))
74-
(catch Exception e
75-
(on-error {:exception e}))))
76-
(fn [e]
77-
(on-error {:exception e})))))
56+
reasoning?* (atom false)
57+
response* (atom nil)
58+
on-error (if on-stream
59+
on-error
60+
(fn [error-data]
61+
(llm-util/log-response logger-tag rid "response-error" body)
62+
(reset! response* error-data)))]
63+
(llm-util/log-request logger-tag rid url body)
64+
@(http/post
65+
url
66+
{:body (json/generate-string body)
67+
:throw-exceptions? false
68+
:async? true
69+
:as (if on-stream :stream :json)}
70+
(fn [{:keys [status body]}]
71+
(try
72+
(if (not= 200 status)
73+
(let [body-str (if on-stream (slurp body) body)]
74+
(logger/warn logger-tag (format "Unexpected response status: %s body: %s" status body-str))
75+
(on-error {:message (format "Ollama response status: %s body: %s" status body-str)}))
76+
(if on-stream
77+
(with-open [rdr (io/reader body)]
78+
(doseq [[event data] (llm-util/event-data-seq rdr)]
79+
(llm-util/log-response logger-tag rid event data)
80+
(on-stream rid event data reasoning?* reason-id)))
81+
(do
82+
(llm-util/log-response logger-tag rid "response" body)
83+
(reset! response*
84+
{:result (:content (:message body))}))))
85+
(catch Exception e
86+
(on-error {:exception e}))))
87+
(fn [e]
88+
(on-error {:exception e})))
89+
@response*))
7890

7991
(defn ^:private ->tools [tools]
8092
(mapv (fn [tool]
@@ -99,21 +111,23 @@
99111
past-messages))
100112

101113
(defn chat! [{:keys [model user-messages reason? instructions api-url past-messages tools]}
102-
{:keys [on-message-received on-error on-prepare-tool-call on-tools-called
103-
on-reason extra-payload]}]
114+
{:keys [on-message-received on-error on-prepare-tool-call on-tools-called
115+
on-reason extra-payload] :as callbacks}]
104116
(let [messages (concat
105117
(normalize-messages (concat [{:role "system" :content instructions}] past-messages))
106118
(normalize-messages user-messages))
119+
stream? (boolean callbacks)
107120
body (deep-merge
108121
{:model model
109122
:messages messages
110123
:think reason?
111124
:tools (->tools tools)
112-
:stream true}
125+
:stream stream?}
113126
extra-payload)
114127
url (format chat-url api-url)
115128
tool-calls* (atom {})
116-
on-response-fn (fn handle-response [rid _event data reasoning?* reason-id]
129+
on-stream-fn (when stream?
130+
(fn handle-stream [rid _event data reasoning?* reason-id]
117131
(let [{:keys [message done_reason]} data]
118132
(cond
119133
(seq (:tool_calls message))
@@ -127,15 +141,15 @@
127141

128142
done_reason
129143
(if-let [tool-call (get @tool-calls* rid)]
130-
;; TODO support multiple tool calls
144+
;; TODO support multiple tool calls
131145
(when-let [{:keys [new-messages]} (on-tools-called [tool-call])]
132146
(swap! tool-calls* dissoc rid)
133147
(base-chat-request!
134148
{:rid (llm-util/gen-rid)
135149
:url url
136150
:body (assoc body :messages (normalize-messages new-messages))
137151
:on-error on-error
138-
:on-response handle-response}))
152+
:on-stream handle-stream}))
139153
(on-message-received {:type :finish
140154
:finish-reason done_reason}))
141155

@@ -155,10 +169,10 @@
155169
:id reason-id})
156170
(reset! reasoning?* false))
157171
(on-message-received {:type :text
158-
:text (:content message)}))))))]
172+
:text (:content message)})))))))]
159173
(base-chat-request!
160174
{:rid (llm-util/gen-rid)
161175
:url url
162176
:body body
163177
:on-error on-error
164-
:on-response on-response-fn})))
178+
:on-stream on-stream-fn})))

‎src/eca/llm_providers/openai.clj‎

Lines changed: 10 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -34,7 +34,9 @@
3434
response* (atom nil)
3535
on-error (if on-stream
3636
on-error
37-
(fn [error-data] (reset! response* error-data)))]
37+
(fn [error-data]
38+
(llm-util/log-response logger-tag rid "response-error" body)
39+
(reset! response* error-data)))]
3840
(llm-util/log-request logger-tag rid url body)
3941
@(http/post
4042
url
@@ -60,11 +62,13 @@
6062
(doseq [[event data] (llm-util/event-data-seq rdr)]
6163
(llm-util/log-response logger-tag rid event data)
6264
(on-stream event data)))
63-
(reset! response*
64-
{:result (reduce
65-
#(str %1 (:text %2))
66-
""
67-
(:content (last (:output body))))})))
65+
(do
66+
(llm-util/log-response logger-tag rid "response" body)
67+
(reset! response*
68+
{:result (reduce
69+
#(str %1 (:text %2))
70+
""
71+
(:content (last (:output body))))}))))
6872
(catch Exception e
6973
(on-error {:exception e}))))
7074
(fn [e]
@@ -245,7 +249,6 @@
245249
:url-relative-path url-relative-path
246250
:api-key api-key
247251
:auth-type auth-type
248-
:stream? stream?
249252
:on-error on-error
250253
:on-stream on-stream-fn})))
251254

‎src/eca/llm_providers/openai_chat.clj‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@
6363
on-error (if on-stream
6464
on-error
6565
(fn [error-data]
66-
(logger/error error-data)
66+
(llm-util/log-response logger-tag rid "response-error" body)
6767
(reset! response* error-data)))]
6868

6969
(llm-util/log-request logger-tag rid url body)

0 commit comments

Comments
 (0)