Skip to content

Commit 4f4b9e9

Browse files
slimslenderslacksslimslenderslacks
committed
Simplify chunk handler
1 parent 38021e4 commit 4f4b9e9

File tree

2 files changed

+96
-85
lines changed

2 files changed

+96
-85
lines changed

‎runbook.md‎

Lines changed: 38 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -20,36 +20,63 @@ bb -m prompts --host-dir /Users/slim/docker/labs-ai-tools-for-devs \
2020

2121
### Running prompts/dockerfiles Conversation Loops
2222

23+
#### test prompts/project_type
24+
2325
Make sure the prompts/project_type prompts work on their own.
2426

2527
```sh
26-
bb -m prompts run /Users/slim/docker/labs-make-runbook jimclark106 darwin prompts/project_type --pat "$(cat ~/.secrets/dockerhub-pat-ai-tools-for-devs.txt)"
28+
bb -m prompts run /Users/slim/docker/labs-make-runbook jimclark106 darwin prompts/project_type --debug
29+
```
30+
31+
```sh
32+
bb -m prompts run /Users/slim/docker/labs-make-runbook jimclark106 darwin prompts/project_type --nostream
33+
```
34+
35+
```sh
36+
bb -m prompts run /Users/slim/docker/labs-make-runbook jimclark106 darwin prompts/project_type --nostream \
37+
--model "llama3.1" \
38+
--url http://localhost:11434/v1/chat/completions
2739
```
2840

41+
#### test prompts/dockerfiles (which uses prompts/project_type)
42+
2943
Now, verify that the prompts/dockerfiles prompts work with `gpt-4`.
3044

3145
```sh
32-
bb -m prompts run /Users/slim/docker/labs-make-runbook jimclark106 darwin prompts/dockerfiles --pat "$(cat ~/.secrets/dockerhub-pat-ai-tools-for-devs.txt)"
46+
bb -m prompts run /Users/slim/docker/labs-make-runbook jimclark106 darwin prompts/dockerfiles
3347
```
3448

3549
Now, let's do the same thing using gpt-4 but without streaming.
3650

3751
```sh
38-
bb -m prompts run /Users/slim/docker/labs-make-runbook jimclark106 darwin prompts/dockerfiles --pat "$(cat ~/.secrets/dockerhub-pat-ai-tools-for-devs.txt)" --nostream --debug
52+
bb -m prompts run /Users/slim/docker/labs-make-runbook jimclark106 darwin prompts/dockerfiles --nostream
3953
```
4054

4155
Now, let's try with llama3.1.
4256

4357
```sh
58+
# docker:command=llama
4459
bb -m prompts run \
4560
--host-dir /Users/slim/docker/labs-make-runbook \
4661
--user jimclark106 \
4762
--platform darwin \
48-
--prompts-dir prompts/dockerfiles \
63+
--prompts-dir prompts/dockerfiles_llama3.1 \
4964
--url http://localhost:11434/v1/chat/completions \
5065
--model "llama3.1" \
51-
--debug \
52-
--nostream
66+
--nostream \
67+
```
68+
69+
Now, let's try with mistral-nemo
70+
71+
```sh
72+
bb -m prompts run \
73+
--host-dir /Users/slim/docker/labs-make-runbook \
74+
--user jimclark106 \
75+
--platform darwin \
76+
--prompts-dir prompts/dockerfiles_mistral-nemo \
77+
--url http://localhost:11434/v1/chat/completions \
78+
--model "mistral-nemo" \
79+
--nostream \
5380
```
5481

5582
Mistral is kind of doing function calls but not openai compatible ones. It's listing a set of functions to call and not getting the arguments correct.
@@ -74,18 +101,7 @@ bb -m prompts run \
74101
--model "llama3-groq-tool-use:latest"
75102
```
76103

77-
78-
```sh
79-
bb -m prompts run \
80-
--host-dir /Users/slim/docker/labs-make-runbook \
81-
--user jimclark106 \
82-
--platform darwin \
83-
--prompts-dir prompts/dockerfiles \
84-
--url http://localhost:11434/v1/chat/completions \
85-
--model "mistral-nemo"
86-
```
87-
88-
### Using Container
104+
#### Using Containerized runner
89105

90106
```sh
91107
docker run --rm \
@@ -97,11 +113,10 @@ docker run --rm \
97113
--mount type=bind,source=$HOME/.openai-api-key,target=/root/.openai-api-key \
98114
vonwig/prompts:local \
99115
run \
100-
/Users/slim/docker/labs-make-runbook \
101-
jimclark106 \
102-
"$(uname -o)" \
103-
local/prompts/dockerfiles \
104-
--pat "$(cat ~/.secrets/dockerhub-pat-ai-tools-for-devs.txt)"
116+
--host-dir /Users/slim/docker/labs-make-runbook \
117+
--user jimclark106 \
118+
--platform "$(uname -o)" \
119+
--prompts-dir local/prompts/dockerfiles
105120
```
106121

107122
### Clean up local images

‎src/openai.clj‎

Lines changed: 58 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -45,25 +45,25 @@
4545
(cb chunk)))
4646
(throw (ex-info "Failed to call OpenAI API" {:body (slurp (:body response))})))))
4747

48-
(defn call-function
48+
(defn call-function
4949
" returns channel that will emit one message and then close"
5050
[function-handler function-name arguments tool-call-id]
5151
(let [c (async/chan)]
5252
(try
5353
(function-handler
54-
function-name
55-
arguments
56-
{:resolve
57-
(fn [output]
58-
(jsonrpc/notify :message {:content (format "\n## ROLE tool (%s)\n%s\n" function-name output)})
59-
(async/go
60-
(async/>! c {:content output :role "tool" :tool_call_id tool-call-id})
61-
(async/close! c)))
62-
:fail (fn [output]
63-
(jsonrpc/notify :message {:content (format "\n## ROLE tool\n function call %s failed %s" function-name output)})
64-
(async/go
65-
(async/>! c {:content output :role "tool" :tool_call_id tool-call-id})
66-
(async/close! c)))})
54+
function-name
55+
arguments
56+
{:resolve
57+
(fn [output]
58+
(jsonrpc/notify :message {:content (format "\n## ROLE tool (%s)\n%s\n" function-name output)})
59+
(async/go
60+
(async/>! c {:content output :role "tool" :tool_call_id tool-call-id})
61+
(async/close! c)))
62+
:fail (fn [output]
63+
(jsonrpc/notify :message {:content (format "\n## ROLE tool\n function call %s failed %s" function-name output)})
64+
(async/go
65+
(async/>! c {:content output :role "tool" :tool_call_id tool-call-id})
66+
(async/close! c)))})
6767
(catch Throwable t
6868
;; function-handlers should handle this on their own but this is just in case
6969
(async/go
@@ -115,37 +115,39 @@
115115
(let [response (atom {})]
116116
(async/go-loop
117117
[]
118-
(let [{:keys [finish-reason] :as e} (async/<! c)]
119-
(cond
120-
(:done e) (let [{calls :tool-calls content :content finish-reason :finish-reason} @response
121-
messages [(merge
122-
{:role "assistant"}
123-
(when (seq (vals calls))
124-
{:tool_calls (->> (vals calls)
125-
(map #(assoc % :type "function")))})
126-
(when content {:content content}))]]
127-
128-
(jsonrpc/notify :functions-done (or (vals calls) ""))
118+
(let [e (async/<! c)]
119+
(if (:done e)
120+
(let [{calls :tool-calls content :content finish-reason :finish-reason} @response
121+
messages [(merge
122+
{:role "assistant"}
123+
(when (seq (vals calls))
124+
{:tool_calls (->> (vals calls)
125+
(map #(assoc % :type "function")))})
126+
(when content {:content content}))]]
127+
128+
(jsonrpc/notify :message {:debug (str @response)})
129+
(jsonrpc/notify :functions-done (or (vals calls) ""))
129130
;; make-tool-calls returns a channel with results of tool call messages
130131
;; so we can continue the conversation
131-
{:finish-reason finish-reason
132-
:messages
133-
(async/<!
134-
(->>
135-
(make-tool-calls
136-
(:tool-handler e)
137-
(vals calls))
138-
(async/reduce conj messages)))})
139-
(:content e) (do
140-
(swap! response update-in [:content] (fnil str "") (:content e))
141-
(when finish-reason (swap! response assoc :finish-reason finish-reason))
142-
(jsonrpc/notify :message {:content (:content e)})
143-
(recur))
144-
:else (let [{:keys [tool_calls finish-reason]} e]
145-
(swap! response update-tool-calls tool_calls)
146-
(when finish-reason (swap! response assoc :finish-reason finish-reason))
147-
(jsonrpc/notify :functions (->> @response :tool-calls vals))
148-
(recur)))))))
132+
{:finish-reason finish-reason
133+
:messages
134+
(async/<!
135+
(->>
136+
(make-tool-calls
137+
(:tool-handler e)
138+
(vals calls))
139+
(async/reduce conj messages)))})
140+
141+
(let [{:keys [content tool_calls finish-reason]} e]
142+
(when content
143+
(swap! response update-in [:content] (fnil str "") content)
144+
(jsonrpc/notify :message {:content content}))
145+
(when tool_calls
146+
(swap! response update-tool-calls tool_calls)
147+
(jsonrpc/notify :functions (->> @response :tool-calls vals)))
148+
(when finish-reason (swap! response assoc :finish-reason finish-reason))
149+
150+
(recur)))))))
149151

150152
(defn parse [s]
151153
(if (= "[DONE]" (string/trim s))
@@ -167,6 +169,9 @@
167169
(some-> chunk
168170
(string/replace #"data: " "")
169171
(parse))]
172+
;; messages will either have a delta, a message, or just a finish_reason,
173+
;; depending on whether it's streaming. Usually, the finish_reason doesn't
174+
;; occur on it's own.
170175
(try
171176
(cond
172177
done? (async/>!!
@@ -176,27 +181,18 @@
176181
(when finish_reason {:finish-reason finish_reason})))
177182

178183
;; streaming
179-
delta (cond
180-
(:content delta) (async/>!! c (merge
181-
{:content (:content delta)}
182-
(when finish_reason {:finish-reason finish_reason})))
183-
184-
(:tool_calls delta) (async/>!! c (merge
185-
delta
186-
(when finish_reason {:finish-reason finish_reason})))
187-
finish_reason (async/>!! c {:finish-reason finish_reason}))
184+
delta
185+
(async/>!! c (merge
186+
delta
187+
(when finish_reason {:finish-reason finish_reason})))
188188

189189
;; non-streaming
190-
message (cond
191-
(:content message) (do (async/>!! c (merge
192-
message
193-
(when finish_reason {:finish-reason finish_reason})))
194-
(async/>!! c {:done true :tool-handler function-handler}))
195-
(:tool_calls message) (do
196-
(async/>!! c (merge
197-
message
198-
(when finish_reason {:finish-reason finish_reason})))
199-
(async/>!! c {:done true :tool-handler function-handler})))
190+
message
191+
(do
192+
(async/>!! c (merge
193+
message
194+
(when finish_reason {:finish-reason finish_reason})))
195+
(async/>!! c {:done true :tool-handler function-handler}))
200196
finish_reason (async/>!! c {:finish-reason finish_reason}))
201197
(catch Throwable _))))]))
202198

0 commit comments

Comments
 (0)