Allow override LLM providers payload

Author: ericdallo

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Fix openai skip streaming response corner cases.
+- Allow override payload of any LLM provider.
 
 ## 0.20.0
 

docs/configuration.md

@@ -156,6 +156,25 @@ Just add to your config the `commands` pointing to `.md` files that will be sear
 }
 ```
 
+## Overriding model payloads
+
+It's possible to override the payload sent to LLMs via `models <modelName> extraPayload` config, this way you can configure custom LLM settings like `temperature`, `reasoning_effort`, `verbosity` etc.
+This config will be merged with current default used by ECA.
+
+Example:
+
+```javascript
+{
+  "models": {
+    "gpt-5": {
+      "extraPayload": {
+        "verbosity": "high"
+      }
+    }
+  }
+}
+```
+
 ## All configs
 
 ### Schema
@@ -191,6 +210,9 @@ interface Config {
         key?: string;
         keyEnv?: string;
     }};
+    models: {[key: string]: {
+      extraPayload: {[key: string]: any}
+    }};
     ollama?: {
         host: string;
         port: string;
@@ -226,6 +248,7 @@ interface Config {
   "mcpTimeoutSeconds" : 10,
   "mcpServers" : [],
   "customProviders": {},
+  "models": {},
   "ollama" : {
     "host" : "http://localhost",
     "port" : 11434,

src/eca/config.clj

@@ -26,6 +26,7 @@
    :disabledTools []
    :mcpTimeoutSeconds 60
    :mcpServers {}
+   :models {}
    :ollama {:host "http://localhost"
             :port 11434
             :useTools true

src/eca/llm_api.clj

@@ -110,6 +110,7 @@
         custom-models (set (mapcat (fn [[k v]]
                                      (map #(str (name k) "/" %) (:models v)))
                                    custom-providers))
+        extra-payload (get-in config [:models (keyword model) :extraPayload])
         callbacks {:on-message-received on-message-received-wrapper
                    :on-error on-error-wrapper
                    :on-prepare-tool-call on-prepare-tool-call-wrapper
@@ -133,6 +134,7 @@
         :past-messages past-messages
         :tools tools
         :web-search web-search
+        :extra-payload extra-payload
         :api-url (openai-api-url)
         :api-key (openai-api-key config)}
        callbacks)
@@ -151,6 +153,7 @@
         :past-messages past-messages
         :tools tools
         :web-search web-search
+        :extra-payload extra-payload
         :api-url (anthropic-api-url)
         :api-key (anthropic-api-key config)}
        callbacks)
@@ -164,7 +167,8 @@
         :instructions instructions
         :user-messages user-messages
         :past-messages past-messages
-        :tools tools}
+        :tools tools
+        :extra-payload extra-payload}
        callbacks)
 
       (contains? custom-models model)
@@ -186,6 +190,7 @@
           :past-messages past-messages
           :web-search web-search
           :tools tools
+          :extra-payload extra-payload
           :api-url url
           :api-key key}
          callbacks))

src/eca/llm_providers/anthropic.clj

@@ -96,22 +96,23 @@
 
 (defn completion!
   [{:keys [model user-messages temperature instructions max-output-tokens
-           api-url api-key reason? reason-tokens past-messages tools web-search]
+           api-url api-key reason? reason-tokens past-messages tools web-search extra-payload]
     :or {temperature 1.0}}
    {:keys [on-message-received on-error on-reason on-prepare-tool-call on-tool-called on-usage-updated]}]
   (let [messages (concat (normalize-messages past-messages)
                          (normalize-messages user-messages))
-        body (assoc-some
-              {:model model
-               :messages (add-cache-to-last-message messages)
-               :max_tokens max-output-tokens
-               :temperature temperature
-               :stream true
-               :tools (->tools tools web-search)
-               :system [{:type "text" :text instructions :cache_control {:type "ephemeral"}}]}
-              :thinking (when (and reason? reason-tokens (> reason-tokens 0))
-                          {:type "enabled"
-                           :budget_tokens reason-tokens}))
+        body (merge (assoc-some
+                     {:model model
+                      :messages (add-cache-to-last-message messages)
+                      :max_tokens max-output-tokens
+                      :temperature temperature
+                      :stream true
+                      :tools (->tools tools web-search)
+                      :system [{:type "text" :text instructions :cache_control {:type "ephemeral"}}]}
+                     :thinking (when (and reason? reason-tokens (> reason-tokens 0))
+                                 {:type "enabled"
+                                  :budget_tokens reason-tokens}))
+                    extra-payload)
 
         on-response-fn
         (fn handle-response [event data content-block* reason-id]

src/eca/llm_providers/ollama.clj

@@ -98,15 +98,16 @@
 
 (defn completion! [{:keys [model user-messages reason? instructions host port past-messages tools]}
                    {:keys [on-message-received on-error on-prepare-tool-call on-tool-called
-                           on-reason]}]
+                           on-reason extra-payload]}]
   (let [messages (concat
                   (normalize-messages (concat [{:role "system" :content instructions}] past-messages))
                   (normalize-messages user-messages))
-        body {:model model
-              :messages messages
-              :think reason?
-              :tools (->tools tools)
-              :stream true}
+        body (merge {:model model
+                     :messages messages
+                     :think reason?
+                     :tools (->tools tools)
+                     :stream true}
+                    extra-payload)
         url (format chat-url (base-url host port))
         tool-calls* (atom {})
         on-response-fn (fn handle-response [rid _event data reasoning?* reason-id]

src/eca/llm_providers/openai.clj

@@ -64,27 +64,28 @@
         past-messages))
 
 (defn completion! [{:keys [model user-messages instructions reason? temperature api-key api-url
-                           max-output-tokens past-messages tools web-search]}
+                           max-output-tokens past-messages tools web-search extra-payload]}
                    {:keys [on-message-received on-error on-prepare-tool-call on-tool-called on-reason on-usage-updated]}]
   (let [input (concat (normalize-messages past-messages)
                       (normalize-messages user-messages))
         tools (cond-> tools
                 web-search (conj {:type "web_search_preview"}))
-        body {:model model
-              :input input
-              :prompt_cache_key (str (System/getProperty "user.name") "@ECA")
-              ;; TODO support parallel
-              :parallel_tool_calls false
-              :instructions instructions
-              ;; TODO allow user specify custom temperature (default 1.0)
-              :temperature temperature
-              :tools tools
-              :reasoning (when reason?
-                           {:effort "medium"
-                            :summary "detailed"})
-              :stream true
-              ;; :verbosity "medium"
-              :max_output_tokens max-output-tokens}
+        body (merge {:model model
+                     :input input
+                     :prompt_cache_key (str (System/getProperty "user.name") "@ECA")
+                     ;; TODO support parallel
+                     :parallel_tool_calls false
+                     :instructions instructions
+                     ;; TODO allow user specify custom temperature (default 1.0)
+                     :temperature temperature
+                     :tools tools
+                     :reasoning (when reason?
+                                  {:effort "medium"
+                                   :summary "detailed"})
+                     :stream true
+                     ;; :verbosity "medium"
+                     :max_output_tokens max-output-tokens}
+                    extra-payload)
         tool-call-by-item-id* (atom {})
         on-response-fn
         (fn handle-response [event data]