Improve ECA prompt to be more precise and output with better quality

ericdallo · ericdallo · commit 3c20fdf0216f · 2025-07-24T17:33:04.000-03:00
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -4,6 +4,7 @@
 
 - Include eca as a  server with tools.
 - Support disable tools via config.
+- Improve ECA prompt to be more precise and output with better quality
 
 ## 0.8.1
 
diff --git a/resources/META-INF/native-image/eca/eca/native-image.properties b/resources/META-INF/native-image/eca/eca/native-image.properties
@@ -6,4 +6,5 @@ Args=-J-Dborkdude.dynaload.aot=true \
      --initialize-at-build-time=com.fasterxml.jackson \
      -H:-CheckToolchain \
      -H:Log=registerResource: \
-     -H:IncludeResources=ECA_VERSION
+     -H:IncludeResources=ECA_VERSION \
+     -H:IncludeResources=eca_prompt.txt
diff --git a/resources/eca_prompt.txt b/resources/eca_prompt.txt
@@ -0,0 +1,18 @@
+You are ECA (Editor Code Assistant), an AI coding assistant that operates on a editor.
+
+You are pair programming with a USER to solve their coding task. Each time the USER sends a message, we may automatically attach some context information about their current state, such as passed contexts, rules defined by USER, project structure, and more. This information may or may not be relevant to the coding task, it is up for you to decide.
+
+{behavior}
+
+<communication>
+The chat is markdown mode.
+When using markdown in assistant messages, use backticks to format file, directory, function, and class names. Use \( and \) for inline math, \[ and \] for block math.
+Pay attention to the langauge name after the code block backticks start, use the full language name like 'javascript' instead of 'js'.
+</communication>
+
+<tool_calling>
+You have tools at your disposal to solve the coding task. Follow these rules regarding tool calls:
+1. ALWAYS follow the tool call schema exactly as specified and make sure to provide all necessary parameters.
+2. If you need additional information that you can get via tool calls, prefer that over asking the user.
+3. If you are not sure about file content or codebase structure pertaining to the user's request, use your tools to read files and gather the relevant information: do NOT guess or make up an answer.
+</tool_calling>
diff --git a/src/eca/features/chat.clj b/src/eca/features/chat.clj
@@ -5,6 +5,7 @@
    [clojure.set :as set]
    [clojure.string :as string]
    [eca.features.index :as f.index]
+   [eca.features.prompt :as f.prompt]
    [eca.features.rules :as f.rules]
    [eca.features.tools :as f.tools]
    [eca.llm-api :as llm-api]
@@ -32,31 +33,6 @@
               "repoMap" [{:type :repoMap}]))
           contexts))
 
-(defn ^:private build-context-str [refined-contexts rules repo-map*]
-  (str
-   "<rules>\n"
-   (reduce
-    (fn [rule-str {:keys [name content]}]
-      (str rule-str (format "<rule name=\"%s\">%s</rule>\n" name content)))
-    ""
-    rules)
-   "</rules>\n"
-   "<contexts>\n"
-   (reduce
-    (fn [context-str {:keys [type path content]}]
-      (str context-str (case type
-                         :file (format "<file path=\"%s\">%s</file>\n" path content)
-                         :repoMap (format "<repoMap description=\"Workspaces structure in a tree view, spaces represent file hierarchy\" >%s</repoMap>" @repo-map*)
-                         "")))
-    ""
-    refined-contexts)
-   "</contexts>"))
-
-(defn ^:private behavior->behavior-str [behavior]
-  (case behavior
-    "chat" "Help with code changes only if user requested/agreed, ask first before do changes, answer questions, and provide explanations."
-    "agent" "Help with code changes when applicable, suggesting you do the changes itself, answer questions, and provide explanations."))
-
 (defn default-model [db config]
   (llm-api/default-model db config))
 
@@ -115,13 +91,11 @@
                                              :state :running
                                              :text "Parsing given context"}))
         db @db*
-        rules (f.rules/all config
-                           (:workspace-folders db)
-                           {:behavior (behavior->behavior-str (or behavior (:chat-default-behavior db)))})
+        rules (f.rules/all config (:workspace-folders db))
         refined-contexts (raw-contexts->refined contexts)
         manual-approval? (get-in config [:toolCall :manualApproval] false)
         repo-map* (delay (f.index/repo-map db {:as-string? true}))
-        context-str (build-context-str refined-contexts rules repo-map*)
+        instructions (f.prompt/build-instructions refined-contexts rules repo-map* (or behavior (:chat-default-behavior db)))
         chosen-model (or model (default-model db config))
         past-messages (get-in db [:chats chat-id :messages] [])
         user-prompt message
@@ -140,7 +114,7 @@
      {:model chosen-model
       :model-config (get-in db [:models chosen-model])
       :user-prompt user-prompt
-      :context context-str
+      :instructions instructions
       :past-messages past-messages
       :config config
       :tools all-tools
diff --git a/src/eca/features/prompt.clj b/src/eca/features/prompt.clj
@@ -0,0 +1,41 @@
+(ns eca.features.prompt
+  (:require
+   [clojure.java.io :as io]
+   [clojure.string :as string]
+   [eca.shared :refer [multi-str]]))
+
+(defn ^:private eca-prompt-template* [] (slurp (io/resource "eca_prompt.txt")))
+
+(def ^:private eca-prompt-template (memoize eca-prompt-template*))
+
+(defn ^:private eca-prompt [behavior]
+  (let [prompt (eca-prompt-template)]
+    (reduce
+     (fn [p [k v]]
+       (string/replace p (str "{" (name k) "}") v))
+     prompt
+     {:behavior (case behavior
+                  "chat" "Answer questions, and provide explanations."
+                  "agent" "You are an agent - please keep going until the user's query is completely resolved, before ending your turn and yielding back to the user. Only terminate your turn when you are sure that the problem is solved. Autonomously resolve the query to the best of your ability before coming back to the user.")})))
+
+(defn build-instructions [refined-contexts rules repo-map* behavior]
+  (multi-str
+   (eca-prompt behavior)
+   "<rules>"
+   (reduce
+    (fn [rule-str {:keys [name content]}]
+      (str rule-str (format "<rule name=\"%s\">%s</rule>\n" name content)))
+    ""
+    rules)
+   "</rules>"
+   ""
+   "<contexts>"
+   (reduce
+    (fn [context-str {:keys [type path content]}]
+      (str context-str (case type
+                         :file (format "<file path=\"%s\">%s</file>\n" path content)
+                         :repoMap (format "<repoMap description=\"Workspaces structure in a tree view, spaces represent file hierarchy\" >%s</repoMap>" @repo-map*)
+                         "")))
+    ""
+    refined-contexts)
+   "</contexts>"))
diff --git a/src/eca/features/rules.clj b/src/eca/features/rules.clj
@@ -2,7 +2,6 @@
   (:require
    [babashka.fs :as fs]
    [clojure.java.io :as io]
-   [clojure.string :as string]
    [eca.config :as config]
    [eca.shared :as shared]))
 
@@ -53,22 +52,7 @@
        (flatten)
        (remove nil?)))
 
-(defn ^:private system-rules []
-  [{:name "ECA System"
-    :type :system
-    :content (str "You are an expert AI coding tool called ECA (Editor Code Assistant). "
-                  "Your behavior is to '<behavior>'. "
-                  "The chat is markdown mode. "
-                  "When responding code blocks, pay attention to use valid markdown languages following Github markdown.")}])
-
-(defn all [config roots variables]
-  (mapv (fn [rule]
-          (reduce
-           (fn [rule [k v]]
-             (update rule :content #(string/replace % (str "<" (name k) ">") v)))
-           rule
-           variables))
-        (concat (system-rules)
-                (config-rules config roots)
-                (global-file-rules)
-                (local-file-rules roots))))
+(defn all [config roots]
+  (concat (config-rules config roots)
+          (global-file-rules)
+          (local-file-rules roots)))
diff --git a/src/eca/llm_api.clj b/src/eca/llm_api.clj
@@ -65,7 +65,7 @@
          :type "function"))
 
 (defn complete!
-  [{:keys [model model-config context user-prompt config on-first-response-received
+  [{:keys [model model-config instructions user-prompt config on-first-response-received
            on-message-received on-error on-prepare-tool-call on-tool-called on-reason
            past-messages tools]}]
   (let [first-response-received* (atom false)
@@ -101,7 +101,7 @@
                    "gpt-4.1"} model)
       (llm-providers.openai/completion!
        {:model model
-        :context context
+        :instructions instructions
         :user-prompt user-prompt
         :past-messages past-messages
         :tools tools
@@ -115,7 +115,7 @@
                    "claude-3-5-haiku-latest"} model)
       (llm-providers.anthropic/completion!
        {:model model
-        :context context
+        :instructions instructions
         :user-prompt user-prompt
         :past-messages past-messages
         :tools tools
@@ -129,7 +129,7 @@
        {:host (-> config :ollama :host)
         :port (-> config :ollama :port)
         :model (string/replace-first model config/ollama-model-prefix "")
-        :context context
+        :instructions instructions
         :user-prompt user-prompt
         :past-messages past-messages
         :tools tools}
@@ -146,7 +146,7 @@
             key (or (:key provider-config) (config/get-env (:keyEnv provider-config)))]
         (provider-fn
          {:model model
-          :context context
+          :instructions instructions
           :user-prompt user-prompt
           :past-messages past-messages
           :tools tools
diff --git a/src/eca/llm_providers/anthropic.clj b/src/eca/llm_providers/anthropic.clj
@@ -76,7 +76,7 @@
    #(assoc-in % [:content 0 :cache_control] {:type "ephemeral"})))
 
 (defn completion!
-  [{:keys [model user-prompt temperature context max-tokens
+  [{:keys [model user-prompt temperature instructions max-tokens
            api-url api-key past-messages tools web-search]
     :or {max-tokens 4096
          temperature 1.0}}
@@ -91,7 +91,7 @@
               ;; TODO support :thinking
               :stream true
               :tools (->tools tools web-search)
-              :system [{:type "text" :text context :cache_control {:type "ephemeral"}}]}
+              :system [{:type "text" :text instructions :cache_control {:type "ephemeral"}}]}
         on-response-fn
         (fn handle-response [event data content-block*]
           (case event
diff --git a/src/eca/llm_providers/ollama.clj b/src/eca/llm_providers/ollama.clj
@@ -75,10 +75,10 @@
              msg))
          past-messages)))
 
-(defn completion! [{:keys [model user-prompt context host port past-messages tools]}
+(defn completion! [{:keys [model user-prompt instructions host port past-messages tools]}
                    {:keys [on-message-received on-error on-prepare-tool-call on-tool-called]}]
   (let [messages (concat
-                  (past-messages->messages past-messages context)
+                  (past-messages->messages past-messages instructions)
                   [{:role "user" :content user-prompt}])
         body {:model model
               :messages messages
@@ -106,7 +106,7 @@
                                  (base-completion-request!
                                   {:rid (llm-util/gen-rid)
                                    :url url
-                                   :body (assoc body :messages (past-messages->messages new-messages context))
+                                   :body (assoc body :messages (past-messages->messages new-messages instructions))
                                    :on-error on-error
                                    :on-response handle-response}))
                                (on-message-received {:type :finish
diff --git a/src/eca/llm_providers/openai.clj b/src/eca/llm_providers/openai.clj
@@ -54,7 +54,7 @@
             msg))
         past-messages))
 
-(defn completion! [{:keys [model user-prompt context temperature api-key api-url past-messages tools web-search]
+(defn completion! [{:keys [model user-prompt instructions temperature api-key api-url past-messages tools web-search]
                     :or {temperature 1.0}}
                    {:keys [on-message-received on-error on-prepare-tool-call on-tool-called on-reason]}]
   (let [input (conj (past-messages->input past-messages)
@@ -64,7 +64,7 @@
         body {:model model
               :input input
               :user (str (System/getProperty "user.name") "@ECA")
-              :instructions context
+              :instructions instructions
               :temperature temperature
               :tools tools
               :stream true}
diff --git a/src/eca/server.clj b/src/eca/server.clj
@@ -13,8 +13,6 @@
 
 (set! *warn-on-reflection* true)
 
-(def ^:private logger-tag "[server]")
-
 (defn ^:private log-wrapper-fn
   [_level & args]
   (apply logger/info args))
diff --git a/src/eca/shared.clj b/src/eca/shared.clj
@@ -38,3 +38,5 @@
          (throw (IllegalArgumentException.
                  "assoc-some expects even number of arguments after map/vector, found odd number")))
        ret))))
+
+(defn multi-str [& strings] (string/join "\n" strings))
diff --git a/test/eca/features/index_test.clj b/test/eca/features/index_test.clj
@@ -4,6 +4,7 @@
    [clojure.java.shell :as shell]
    [clojure.test :refer [deftest is testing]]
    [eca.features.index :as f.index]
+   [eca.shared :refer [multi-str]]
    [eca.test-helper :as h]
    [matcher-combinators.test :refer [match?]]))
 
@@ -43,9 +44,10 @@
            (eca.features.index/repo-map {:workspace-folders [{:uri (h/file-uri "file:///fake/repo")}]})))))
   (testing "returns string tree with as-string? true"
     (with-redefs [f.index/git-ls-files (constantly ["foo.clj" "bar/baz.clj"])]
-      (is (= (str (h/file-path "/fake/repo") "\n"
-                  " bar\n"
-                  "  baz.clj\n"
-                  " foo.clj\n")
+      (is (= (multi-str (h/file-path "/fake/repo")
+                        " bar"
+                        "  baz.clj"
+                        " foo.clj"
+                        "")
              (eca.features.index/repo-map {:workspace-folders [{:uri (h/file-uri "file:///fake/repo")}]}
                                           {:as-string? true}))))))
diff --git a/test/eca/features/prompt_test.clj b/test/eca/features/prompt_test.clj
@@ -0,0 +1,24 @@
+(ns eca.features.prompt-test
+  (:require
+   [clojure.test :refer [deftest is testing]]
+   [eca.features.prompt :as prompt]
+   [clojure.string :as string]))
+
+(deftest build-instructions-test
+  (testing "Should create instructions with rules, contexts, and behavior"
+    (let [refined-contexts [{:type :file :path "foo.clj" :content "(ns foo)"}
+                            {:type :repoMap :path nil :content nil}]
+          rules [{:name "rule1" :content "First rule"}
+                 {:name "rule2" :content "Second rule"}]
+          fake-repo-map (delay "TREE")
+          behavior "agent"
+          result (prompt/build-instructions refined-contexts rules fake-repo-map behavior)]
+      (is (string/includes? result "You are ECA"))
+      (is (string/includes? result "<rules>"))
+      (is (string/includes? result "<rule name=\"rule1\">First rule</rule>"))
+      (is (string/includes? result "<rule name=\"rule2\">Second rule</rule>"))
+      (is (string/includes? result "<contexts>"))
+      (is (string/includes? result "<file path=\"foo.clj\">(ns foo)</file>"))
+      (is (string/includes? result "<repoMap description=\"Workspaces structure in a tree view, spaces represent file hierarchy\" >TREE</repoMap>"))
+      (is (string/includes? result "</contexts>"))
+      (is (string? result)))))
diff --git a/test/eca/features/rules_test.clj b/test/eca/features/rules_test.clj
