Return only authenticated models

Author: ericdallo

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Support `chat/selectedBehaviorChanged` client notification.
+- Update models according with supported models given its auth or key/url configuration.
 
 ## 0.42.0
 

docs/models.md

@@ -1,9 +1,11 @@
 # Models
 
-All providers and models are configured under `providers` config.
+All providers and models are configured under `providers` config or via `/login` when supported.
 
 Models capabilities and configurations are retrieved from [models.dev](https://models.dev) API.
 
+ECA will return to clients the models configured, either via config or login.
+
 ## Built-in providers and capabilities
 
 | model                         | tools (MCP) | reasoning / thinking | prompt caching | web_search |

integration-test/entrypoint.clj

@@ -6,12 +6,11 @@
 
 (def namespaces
   '[integration.initialize-test
-    ;integration.chat.openai-test
-    ;integration.chat.anthropic-test
-    ;integration.chat.github-copilot-test
-    ;integration.chat.ollama-test
-    ;integration.chat.custom-provider-test
-    ])
+    integration.chat.openai-test
+    integration.chat.anthropic-test
+    integration.chat.github-copilot-test
+    integration.chat.ollama-test
+    integration.chat.custom-provider-test])
 
 (defn timeout [timeout-ms callback]
   (let [fut (future (callback))

integration-test/integration/fixture.clj

@@ -7,17 +7,20 @@
 (def base-llm-mock-url
   (str "http://localhost:" llm-mock.server/port))
 
+(def default-providers
+  {"openai" {:url (str base-llm-mock-url "/openai")
+             :key "foo-key"
+             :keyEnv "FOO"}
+   "anthropic" {:url (str base-llm-mock-url "/anthropic")
+                :key "foo-key"
+                :keyEnv "FOO"}
+   "github-copilot" {:url (str base-llm-mock-url "/github-copilot")
+                     :key "foo-key"
+                     :keyEnv "FOO"}})
+
 (def default-init-options {:pureConfig true
                            :toolCall {:approval {:byDefault "allow"}}
-                           :providers {"openai" {:url (str base-llm-mock-url "/openai")
-                                                 :key "foo-key"
-                                                 :keyEnv "FOO"}
-                                       "anthropic" {:url (str base-llm-mock-url "/anthropic")
-                                                    :key "foo-key"
-                                                    :keyEnv "FOO"}
-                                       "github-copilot" {:url (str base-llm-mock-url "/github-copilot")
-                                                         :key "foo-key"
-                                                         :keyEnv "FOO"}}})
+                           :providers default-providers})
 
 (defn initialize-request
   ([]

integration-test/integration/initialize_test.clj

@@ -67,35 +67,48 @@
 
 (deftest initialize-with-custom-providers
   (eca/start-process!)
-  (testing "initialize request with custom providers"
-    (is (match?
-         {:models ["anthropic/claude-3-5-haiku-20241022"
-                   "anthropic/claude-opus-4-1-20250805"
-                   "anthropic/claude-opus-4-20250514"
-                   "anthropic/claude-sonnet-4-20250514"
-                   "github-copilot/claude-sonnet-4"
-                   "github-copilot/gemini-2.5-pro"
-                   "github-copilot/gpt-4.1"
-                   "github-copilot/gpt-5"
-                   "github-copilot/gpt-5-mini"
-                   "my-custom/bar2"
-                   "my-custom/foo1"
-                   "openai/gpt-4.1"
-                   "openai/gpt-5"
-                   "openai/gpt-5-mini"
-                   "openai/gpt-5-nano"
-                   "openai/o3"
-                   "openai/o4-mini"]
-          :chatDefaultModel "myCustom/bar-2"
-          :chatBehaviors ["agent" "plan"]
-          :chatDefaultBehavior "agent"
-          :chatWelcomeMessage "Welcome to ECA!\n\nType '/' for commands\n\n"}
-         (eca/request! (fixture/initialize-request
-                        {:initializationOptions (merge fixture/default-init-options
-                                                       {:defaultModel "myCustom/bar-2"
-                                                        :providers
-                                                        {"myCustom" {:api "openai"
-                                                                     :urlEnv "MY_CUSTOM_API_URL"
-                                                                     :keyEnv "MY_CUSTOM_API_KEY"
-                                                                     :models {"foo1" {}
-                                                                              "bar2" {}}}}})}))))))
+  (let [models ["anthropic/claude-3-5-haiku-20241022"
+                "anthropic/claude-opus-4-1-20250805"
+                "anthropic/claude-opus-4-20250514"
+                "anthropic/claude-sonnet-4-20250514"
+                "github-copilot/claude-sonnet-4"
+                "github-copilot/gemini-2.5-pro"
+                "github-copilot/gpt-4.1"
+                "github-copilot/gpt-5"
+                "github-copilot/gpt-5-mini"
+                "my-custom/bar2"
+                "my-custom/foo1"
+                "openai/gpt-4.1"
+                "openai/gpt-5"
+                "openai/gpt-5-mini"
+                "openai/gpt-5-nano"
+                "openai/o3"
+                "openai/o4-mini"]]
+    (testing "initialize request with custom providers"
+      (is (match?
+           {:models models
+            :chatDefaultModel "my-custom/bar-2"
+            :chatBehaviors ["agent" "plan"]
+            :chatDefaultBehavior "agent"
+            :chatWelcomeMessage "Welcome to ECA!\n\nType '/' for commands\n\n"}
+           (eca/request! (fixture/initialize-request
+                          {:initializationOptions (merge fixture/default-init-options
+                                                         {:defaultModel "my-custom/bar-2"
+                                                          :providers
+                                                          (merge fixture/default-providers
+                                                                 {"my-custom" {:api "openai-chat"
+                                                                              :url "MY_URL"
+                                                                              :key "MY_KEY"
+                                                                              :models {"foo1" {}
+                                                                                       "bar2" {}}}})})})))))
+    (testing "initialized notification"
+      (eca/notify! (fixture/initialized-notification)))
+
+    (testing "config updated"
+      (is (match?
+           {:chat {:models models
+                   :defaultModel "my-custom/bar-2"
+                   :behaviors ["agent" "plan"]
+                   :defaultBehavior "agent"
+                   :welcomeMessage "Welcome to ECA!\n\nType '/' for commands\n\n"}}
+           (eca/client-awaits-server-notification :config/updated))))))

src/eca/db.clj

@@ -104,11 +104,14 @@
     (when (= version (:version cache))
       cache)))
 
-(defn load-db-from-cache! [db*]
-  (when-let [global-cache (read-global-cache)]
-    (swap! db* shared/deep-merge global-cache))
-  (when-let [global-by-workspace-cache (read-global-by-workspaces-cache (:workspace-folders @db*))]
-    (swap! db* shared/deep-merge global-by-workspace-cache)))
+(defn load-db-from-cache! [db* config]
+  (when-not (:pureConfig config)
+    (when-let [global-cache (read-global-cache)]
+      (logger/info logger-tag "Loading from global-cache caches...")
+      (swap! db* shared/deep-merge global-cache))
+    (when-let [global-by-workspace-cache (read-global-by-workspaces-cache (:workspace-folders @db*))]
+      (logger/info logger-tag "Loading from workspace-cache caches...")
+      (swap! db* shared/deep-merge global-by-workspace-cache))))
 
 (defn ^:private normalize-db-for-workspace-write [db]
   (-> (select-keys db [:chats])

src/eca/features/login.clj

@@ -2,7 +2,8 @@
   (:require
    [clojure.string :as string]
    [eca.db :as db]
-   [eca.messenger :as messenger]))
+   [eca.messenger :as messenger]
+   [eca.models :as models]))
 
 (defmulti login-step (fn [ctx] [(:provider ctx) (:step ctx)]))
 
@@ -14,14 +15,16 @@
                :provider provider
                :db* db*}))
 
-(defn continue [{:keys [message chat-id request-id]} db* messenger]
+(defn continue [{:keys [message chat-id request-id]} db* messenger config]
   (let [provider (get-in @db* [:chats chat-id :login-provider])
         step (get-in @db* [:auth provider :step])
         input (string/trim message)
         ctx {:chat-id chat-id
              :step step
              :input input
              :db* db*
+             :config config
+             :messenger messenger
              :provider provider
              :send-msg! (fn [msg]
                           (messenger/chat-content-received
@@ -56,3 +59,12 @@
     :step :login/renew-token
     :db* db*})
   (db/update-global-cache! @db*))
+
+(defn login-done! [{:keys [db* config messenger]}]
+  (models/sync-models! db*
+                       config
+                       (fn [new-models]
+                         (messenger/config-updated
+                          messenger
+                          {:chat
+                           {:models (sort (keys new-models))}}))))

src/eca/handlers.clj

@@ -1,57 +1,18 @@
 (ns eca.handlers
   (:require
-   [clojure.string :as string]
    [eca.config :as config]
    [eca.db :as db]
    [eca.features.chat :as f.chat]
    [eca.features.login :as f.login]
    [eca.features.tools :as f.tools]
    [eca.features.tools.mcp :as f.mcp]
-   [eca.llm-api :as llm-api]
    [eca.logger :as logger]
    [eca.messenger :as messenger]
    [eca.models :as models]
    [eca.shared :as shared]))
 
 (set! *warn-on-reflection* true)
 
-(defn ^:private sync-models! [db* config on-models-updated]
-  (let [all-models (models/all)
-        all-models (reduce
-                    (fn [p [provider provider-config]]
-                      (merge p
-                             (reduce
-                              (fn [m [model _model-config]]
-                                (let [full-model (str provider "/" model)
-                                      model-capabilities (merge
-                                                          (or (get all-models full-model)
-                                                                     ;; we guess the capabilities from
-                                                                     ;; the first model with same name
-                                                              (when-let [found-full-model (first (filter #(= (shared/normalize-model-name model)
-                                                                                                             (shared/normalize-model-name (second (string/split % #"/" 2))))
-                                                                                                         (keys all-models)))]
-                                                                (get all-models found-full-model))
-                                                              {:tools true
-                                                               :reason? true
-                                                               :web-search true}))]
-                                  (assoc m full-model model-capabilities)))
-                              {}
-                              (:models provider-config))))
-                    {}
-                    (:providers config))
-        all-models (if-let [local-models (seq (llm-api/local-models config))]
-                     (let [models (reduce
-                                   (fn [models {:keys [model] :as ollama-model}]
-                                     (assoc models
-                                            (str config/ollama-model-prefix model)
-                                            (select-keys ollama-model [:tools :reason?])))
-                                   {}
-                                   local-models)]
-                       (merge all-models models))
-                     all-models)]
-    (swap! db* assoc :models all-models)
-    (on-models-updated)))
-
 (defn initialize [{:keys [db*]} params]
   (logger/logging-task
    :eca/initialize
@@ -63,12 +24,12 @@
             :workspace-folders (:workspace-folders params)
             :client-capabilities (:capabilities params))
      (when-not (:pureConfig config)
-       (db/load-db-from-cache! db*))
+       (db/load-db-from-cache! db* config))
 
      ;; Deprecated
      ;; For backward compatibility,
      ;; we now return chat config via `config/updated` notification.
-     (sync-models! db* config (fn []))
+     (models/sync-models! db* config (fn [_]))
      (let [db @db*]
        {:models (sort (keys (:models db)))
         :chat-default-model (f.chat/default-model db config)
@@ -83,11 +44,11 @@
                                   (let [new-providers-hash (hash (:providers config))]
                                     (when (not= (:providers-config-hash @db*) new-providers-hash)
                                       (swap! db* assoc :providers-config-hash new-providers-hash)
-                                      (sync-models! db* config (fn []
+                                      (models/sync-models! db* config (fn [models]
                                                                  (let [db @db*]
                                                                    (config/notify-fields-changed-only!
                                                                     {:chat
-                                                                     {:models (sort (keys (:models db)))
+                                                                     {:models (sort (keys models))
                                                                       :default-model (f.chat/default-model db config)
                                                                       :behaviors (:chat-behaviors db)
                                                                       :default-behavior (or (:defaultBehavior (:chat config)) ;;legacy
@@ -122,7 +83,7 @@
   (logger/logging-task
    :eca/chat-prompt
    (case (get-in @db* [:chats (:chat-id params) :status])
-     :login (f.login/continue params db* messenger)
+     :login (f.login/continue params db* messenger config)
      (f.chat/prompt params db* messenger config))))
 
 (defn chat-query-context [{:keys [db* config]} params]

src/eca/llm_api.clj

@@ -8,6 +8,7 @@
    [eca.llm-providers.ollama :as llm-providers.ollama]
    [eca.llm-providers.openai :as llm-providers.openai]
    [eca.llm-providers.openai-chat :as llm-providers.openai-chat]
+   [eca.llm-util :as llm-util]
    [eca.logger :as logger]))
 
 (set! *warn-on-reflection* true)
@@ -32,25 +33,6 @@
           (string/join "\n" (subvec lines start end)))
         content))))
 
-(defn ^:private provider-api-key [provider provider-auth config]
-  (or (get-in config [:providers (name provider) :key])
-      (:api-key provider-auth)
-      (some-> (get-in config [:providers (name provider) :keyEnv]) config/get-env)))
-
-(defn ^:private provider-api-url [provider config]
-  (or (get-in config [:providers (name provider) :url])
-      (some-> (get-in config [:providers (name provider) :urlEnv]) config/get-env)))
-
-(defn local-models [config]
-  (let [ollama-api-url (provider-api-url "ollama" config)]
-    (mapv
-     (fn [{:keys [model] :as ollama-model}]
-       (let [capabilities (llm-providers.ollama/model-capabilities {:api-url ollama-api-url :model model})]
-         (assoc ollama-model
-                :tools (boolean (some #(= % "tools") capabilities))
-                :reason? (boolean (some #(= % "thinking") capabilities)))))
-     (llm-providers.ollama/list-models {:api-url ollama-api-url}))))
-
 (defn default-model
   "Returns the default LLM model checking this waterfall:
   - defaultModel set
@@ -63,9 +45,9 @@
   (let [[decision model]
         (or (when-let [config-default-model (:defaultModel config)]
               [:config-default-model config-default-model])
-            (when (provider-api-key "anthropic" (get-in db [:auth "anthropic"]) config)
+            (when (llm-util/provider-api-key "anthropic" (get-in db [:auth "anthropic"]) config)
               [:api-key-found "anthropic/claude-sonnet-4-20250514"])
-            (when (provider-api-key "openai" (get-in db [:auth "openai"]) config)
+            (when (llm-util/provider-api-key "openai" (get-in db [:auth "openai"]) config)
               [:api-key-found "openai/gpt-5"])
             (when (get-in db [:auth "github-copilot" :api-key])
               [:api-key-found "github-copilot/gpt-4.1"])
@@ -109,8 +91,8 @@
         provider-config (get-in config [:providers provider])
         model-config (get-in provider-config [:models model])
         extra-payload (:extraPayload model-config)
-        api-key (provider-api-key provider provider-auth config)
-        api-url (provider-api-url provider config)
+        api-key (llm-util/provider-api-key provider provider-auth config)
+        api-url (llm-util/provider-api-url provider config)
         provider-auth-type (:type provider-auth)
         callbacks {:on-message-received on-message-received-wrapper
                    :on-error on-error-wrapper