Merge pull request #180 from editor-code-assistant/non-stream-support

Non stream support

Author: ericdallo

CHANGELOG.md

@@ -4,6 +4,7 @@
 
 - Improved `eca_edit_file` to automatically handle whitespace and indentation differences in single-occurrence edits.
 - Fix contexts in user prompts (not system contexts) not parsing lines ranges properly.
+- Support non-stream providers on openai-chat API. #174
 
 ## 0.73.5
 

src/eca/features/chat.clj

@@ -93,140 +93,138 @@
                      :on-tools-called on-tools-called
                      :on-reason on-reason
                      :on-usage-updated on-usage-updated})]
-    ;; We spawn a new future to not block the lsp4clj thread
-    ;; in case a tool call approval is needed
-    (future
-      (try
-        (when-not api-url (throw (ex-info (format "API url not found.\nMake sure you have provider '%s' configured properly." provider) {})))
-        (cond
-          (= "openai" provider)
-          (llm-providers.openai/create-response!
-           {:model real-model
-            :instructions instructions
-            :user-messages user-messages
-            :max-output-tokens max-output-tokens
-            :reason? reason?
-            :supports-image? supports-image?
-            :past-messages past-messages
-            :tools tools
-            :web-search web-search
-            :extra-payload (merge {:parallel_tool_calls true}
-                                  extra-payload)
-            :api-url api-url
-            :api-key api-key
-            :auth-type auth-type}
-           callbacks)
+    (try
+      (when-not api-url (throw (ex-info (format "API url not found.\nMake sure you have provider '%s' configured properly." provider) {})))
+      (cond
+        (= "openai" provider)
+        (llm-providers.openai/create-response!
+         {:model real-model
+          :instructions instructions
+          :user-messages user-messages
+          :max-output-tokens max-output-tokens
+          :reason? reason?
+          :supports-image? supports-image?
+          :past-messages past-messages
+          :tools tools
+          :web-search web-search
+          :extra-payload (merge {:parallel_tool_calls true}
+                                extra-payload)
+          :api-url api-url
+          :api-key api-key
+          :auth-type auth-type}
+         callbacks)
 
-          (= "anthropic" provider)
-          (llm-providers.anthropic/chat!
-           {:model real-model
-            :instructions instructions
-            :user-messages user-messages
-            :max-output-tokens max-output-tokens
-            :reason? reason?
-            :supports-image? supports-image?
-            :past-messages past-messages
-            :tools tools
-            :web-search web-search
-            :extra-payload extra-payload
-            :api-url api-url
-            :api-key api-key
-            :auth-type auth-type}
-           callbacks)
+        (= "anthropic" provider)
+        (llm-providers.anthropic/chat!
+         {:model real-model
+          :instructions instructions
+          :user-messages user-messages
+          :max-output-tokens max-output-tokens
+          :reason? reason?
+          :supports-image? supports-image?
+          :past-messages past-messages
+          :tools tools
+          :web-search web-search
+          :extra-payload extra-payload
+          :api-url api-url
+          :api-key api-key
+          :auth-type auth-type}
+         callbacks)
 
-          (= "github-copilot" provider)
-          (llm-providers.openai-chat/chat-completion!
-           {:model real-model
-            :instructions instructions
-            :user-messages user-messages
-            :max-output-tokens max-output-tokens
-            :reason? reason?
-            :supports-image? supports-image?
-            :past-messages past-messages
-            :tools tools
-            :extra-payload (merge {:parallel_tool_calls true}
-                                  extra-payload)
-            :api-url api-url
-            :api-key api-key
-            :extra-headers {"openai-intent" "conversation-panel"
-                            "x-request-id" (str (random-uuid))
-                            "vscode-sessionid" ""
-                            "vscode-machineid" ""
-                            "Copilot-Vision-Request" "true"
-                            "copilot-integration-id" "vscode-chat"}}
-           callbacks)
+        (= "github-copilot" provider)
+        (llm-providers.openai-chat/chat-completion!
+         {:model real-model
+          :instructions instructions
+          :user-messages user-messages
+          :max-output-tokens max-output-tokens
+          :reason? reason?
+          :supports-image? supports-image?
+          :past-messages past-messages
+          :tools tools
+          :extra-payload (merge {:parallel_tool_calls true}
+                                extra-payload)
+          :api-url api-url
+          :api-key api-key
+          :extra-headers {"openai-intent" "conversation-panel"
+                          "x-request-id" (str (random-uuid))
+                          "vscode-sessionid" ""
+                          "vscode-machineid" ""
+                          "Copilot-Vision-Request" "true"
+                          "copilot-integration-id" "vscode-chat"}}
+         callbacks)
+
+        (= "google" provider)
+        (llm-providers.openai-chat/chat-completion!
+         {:model real-model
+          :instructions instructions
+          :user-messages user-messages
+          :max-output-tokens max-output-tokens
+          :reason? reason?
+          :supports-image? supports-image?
+          :past-messages past-messages
+          :tools tools
+          :thinking-tag "thought"
+          :extra-payload (merge {:parallel_tool_calls false}
+                                (when reason?
+                                  {:extra_body {:google {:thinking_config {:include_thoughts true}}}})
+                                extra-payload)
+          :api-url api-url
+          :api-key api-key}
+         callbacks)
+
+        (= "ollama" provider)
+        (llm-providers.ollama/chat!
+         {:api-url api-url
+          :reason? (:reason? model-capabilities)
+          :supports-image? supports-image?
+          :model real-model
+          :instructions instructions
+          :user-messages user-messages
+          :past-messages past-messages
+          :tools tools
+          :extra-payload extra-payload}
+         callbacks)
 
-          (= "google" provider)
-          (llm-providers.openai-chat/chat-completion!
+        model-config
+        (let [provider-fn (case (:api provider-config)
+                            ("openai-responses"
+                             "openai") llm-providers.openai/create-response!
+                            "anthropic" llm-providers.anthropic/chat!
+                            "openai-chat" llm-providers.openai-chat/chat-completion!
+                            (on-error {:message (format "Unknown model %s for provider %s" (:api provider-config) provider)}))
+              url-relative-path (:completionUrlRelativePath provider-config)]
+          (provider-fn
            {:model real-model
             :instructions instructions
             :user-messages user-messages
             :max-output-tokens max-output-tokens
+            :web-search web-search
             :reason? reason?
             :supports-image? supports-image?
             :past-messages past-messages
             :tools tools
-            :thinking-tag "thought"
-            :extra-payload (merge {:parallel_tool_calls false}
-                                  (when reason?
-                                    {:extra_body {:google {:thinking_config {:include_thoughts true}}}})
-                                  extra-payload)
+            :extra-payload extra-payload
+            :url-relative-path url-relative-path
             :api-url api-url
             :api-key api-key}
-           callbacks)
+           callbacks))
 
-          (= "ollama" provider)
-          (llm-providers.ollama/chat!
-           {:api-url api-url
-            :reason? (:reason? model-capabilities)
-            :supports-image? supports-image?
-            :model real-model
-            :instructions instructions
-            :user-messages user-messages
-            :past-messages past-messages
-            :tools tools
-            :extra-payload extra-payload}
-           callbacks)
-
-          model-config
-          (let [provider-fn (case (:api provider-config)
-                              ("openai-responses"
-                               "openai") llm-providers.openai/create-response!
-                              "anthropic" llm-providers.anthropic/chat!
-                              "openai-chat" llm-providers.openai-chat/chat-completion!
-                              (on-error {:message (format "Unknown model %s for provider %s" (:api provider-config) provider)}))
-                url-relative-path (:completionUrlRelativePath provider-config)]
-            (provider-fn
-             {:model real-model
-              :instructions instructions
-              :user-messages user-messages
-              :max-output-tokens max-output-tokens
-              :web-search web-search
-              :reason? reason?
-              :supports-image? supports-image?
-              :past-messages past-messages
-              :tools tools
-              :extra-payload extra-payload
-              :url-relative-path url-relative-path
-              :api-url api-url
-              :api-key api-key}
-             callbacks))
+        :else
+        (on-error {:message (format "ECA Unsupported model %s for provider %s" real-model provider)}))
+      (catch Exception e
+        (on-error {:exception e})))))
 
-          :else
-          (on-error {:message (format "ECA Unsupported model %s for provider %s" real-model provider)}))
-        (catch Exception e
-          (on-error {:exception e}))))))
-
-(defn async-prompt! [{:keys [provider model model-capabilities instructions user-messages config on-first-response-received
-                             on-message-received on-error on-prepare-tool-call on-tools-called on-reason on-usage-updated
-                             past-messages tools provider-auth]
-                      :or {on-first-response-received identity
-                           on-message-received identity
-                           on-error identity
-                           on-prepare-tool-call identity
-                           on-tools-called identity
-                           on-reason identity
-                           on-usage-updated identity}}]
+(defn sync-or-async-prompt!
+  [{:keys [provider model model-capabilities instructions user-messages config on-first-response-received
+           on-message-received on-error on-prepare-tool-call on-tools-called on-reason on-usage-updated
+           past-messages tools provider-auth]
+    :or {on-first-response-received identity
+         on-message-received identity
+         on-error identity
+         on-prepare-tool-call identity
+         on-tools-called identity
+         on-reason identity
+         on-usage-updated identity}}]
   (let [first-response-received* (atom false)
         emit-first-message-fn (fn [& args]
                                 (when-not @first-response-received*
@@ -244,38 +242,73 @@
         on-error-wrapper (fn [{:keys [exception] :as args}]
                            (when-not (:silent? (ex-data exception))
                              (logger/error args)
-                             (on-error args)))]
-    (prompt!
-     {:sync? false
-      :provider provider
-      :model model
-      :model-capabilities model-capabilities
-      :instructions instructions
-      :tools tools
-      :provider-auth provider-auth
-      :past-messages past-messages
-      :user-messages user-messages
-      :on-message-received on-message-received-wrapper
-      :on-prepare-tool-call on-prepare-tool-call-wrapper
-      :on-tools-called on-tools-called
-      :on-usage-updated on-usage-updated
-      :on-reason on-reason-wrapper
-      :on-error on-error-wrapper
-      :config config})))
+                             (on-error args)))
+        provider-config (get-in config [:providers provider])
+        model-config (get-in provider-config [:models model])
+        extra-payload (:extraPayload model-config)
+        stream? (if (not (nil? (:stream extra-payload)))
+                  (:stream extra-payload)
+                  true)]
+    (if (not stream?)
+      (loop [result (prompt!
+                     {:sync? true
+                      :provider provider
+                      :model model
+                      :model-capabilities model-capabilities
+                      :instructions instructions
+                      :tools tools
+                      :provider-auth provider-auth
+                      :past-messages past-messages
+                      :user-messages user-messages
+                      :on-error on-error-wrapper
+                      :config config})]
+        (let [{:keys [error output-text reason-text tools-to-call call-tools-fn reason-id usage]} result]
+          (if error
+            (on-error-wrapper error)
+            (do
+              (when reason-text
+                (on-reason-wrapper {:status :started :id reason-id})
+                (on-reason-wrapper {:status :thinking :id reason-id :text reason-text})
+                (on-reason-wrapper {:status :finished :id reason-id}))
+              (on-message-received-wrapper {:type :text :text output-text})
+              (some-> usage (on-usage-updated))
+              (if-let [new-result (when (seq tools-to-call)
+                                    (doseq [tool-to-call tools-to-call]
+                                      (on-prepare-tool-call tool-to-call))
+                                    (call-tools-fn on-tools-called))]
+                (recur new-result)
+                (on-message-received-wrapper {:type :finish :finish-reason "stop"}))))))
+      (prompt!
+       {:sync? false
+        :provider provider
+        :model model
+        :model-capabilities model-capabilities
+        :instructions instructions
+        :tools tools
+        :provider-auth provider-auth
+        :past-messages past-messages
+        :user-messages user-messages
+        :on-message-received on-message-received-wrapper
+        :on-prepare-tool-call on-prepare-tool-call-wrapper
+        :on-tools-called on-tools-called
+        :on-usage-updated on-usage-updated
+        :on-reason on-reason-wrapper
+        :on-error on-error-wrapper
+        :config config}))))
 
 (defn sync-prompt!
   [{:keys [provider model model-capabilities instructions
            prompt past-messages user-messages config tools provider-auth]}]
-  @(prompt!
-    {:sync? true
-     :provider provider
-     :model model
-     :model-capabilities model-capabilities
-     :instructions instructions
-     :tools tools
-     :provider-auth provider-auth
-     :past-messages past-messages
-     :user-messages (or user-messages
-                        [{:role "user" :content [{:type :text :text prompt}]}])
-     :config config
-     :on-error (fn [error] {:error error})}))
+  (prompt!
+   {:sync? true
+    :provider provider
+    :model model
+    :model-capabilities model-capabilities
+    :instructions instructions
+    :tools tools
+    :provider-auth provider-auth
+    :past-messages past-messages
+    :user-messages (or user-messages
+                       [{:role "user" :content [{:type :text :text prompt}]}])
+    :config config
+    :on-error (fn [error] {:error error})}))

src/eca/llm_api.clj

@@ -103,7 +103,7 @@
               (do
                 (llm-util/log-response logger-tag rid "response" body)
                 (reset! response*
-                        {:result (:text (last (:content body)))}))))
+                        {:output-text (:text (last (:content body)))}))))
           (catch Exception e
             (on-error {:exception e}))))
       (fn [e]

src/eca/llm_providers/anthropic.clj

@@ -81,7 +81,7 @@
               (do
                 (llm-util/log-response logger-tag rid "response" body)
                 (reset! response*
-                        {:result (:content (:message body))}))))
+                        {:output-text (:content (:message body))}))))
           (catch Exception e
             (on-error {:exception e}))))
       (fn [e]