Support repo-map context

Author: ericdallo

CHANGELOG.md

@@ -3,6 +3,7 @@
 ## Unreleased
 
 - Support tool call approval and configuration to manual approval.
+- Initial support for repo-map context.
 
 ## 0.7.0
 

dev/user.clj

@@ -0,0 +1,12 @@
+(ns user
+  (:require
+   [eca.db :as db]))
+
+(alter-var-root #'*warn-on-reflection* (constantly true))
+
+(def ^:dynamic *db* @db/db*)
+
+#_{:clojure-lsp/ignore [:clojure-lsp/unused-public-var]}
+(defmacro with-workspace-root [root-uri & body]
+  `(binding [*db* (assoc *db* :workspace-folders [{:name "dev" :uri ~root-uri}])]
+     ~@body))

docs/features.md

@@ -33,6 +33,15 @@ Provides access to run shell commands, useful to run build tools, tests, and oth
 
 - `eca_shell_command`: run shell command. Supports configs to exclude commands via `:nativeTools :shell :excludeCommands`.
 
+### Contexts
+
+User can include contexts to the chat, which can help LLM generate output with better quality.
+Here are the current supported contexts types:
+
+- `file`: a file in the workspace, server will pass its content to LLM.
+- `directoryu`: a directory in the workspace, server will read all file contexts and pass to LLM.
+- `repoMap`: a summary view of workspaces files and folders, server will calculate this and pass to LLM. Currently, the repo-map includes only the file paths in git.
+
 ##  Completion
 
 Soon

docs/protocol.md

@@ -278,7 +278,7 @@ type ChatModel =
  */
 type OllamaRunningModel = string
 
-type ChatContext = FileContext | DirectoryContext | WebContext | CodeContext;
+type ChatContext = FileContext | DirectoryContext | WebContext | RepoMapContext;
 
 /**
  * Context related to a file in the workspace
@@ -314,23 +314,13 @@ interface WebContext {
 }
 
 /**
- * Context related to code snippets
+ * Context about the workspaces repo-map, automatically calculated by server.
+ * Clients should include this to chat by default but users may want exclude 
+ * this context to reduce context size if needed.
  */
-interface CodeContext {
-    type: 'code';
-    /**
-     * The code content
-     */
-    content: string;
-    /**
-     * The programming language of the code
-     */
-    language: string;
-    /**
-     * Name or description of what this code represents
-     */
-    description?: string;
-}
+interface RepoMapContext {
+    type: 'repoMap'; 
+ }
 ```
 
 _Response:_

src/eca/features/chat.clj

@@ -1,23 +1,33 @@
 (ns eca.features.index
   (:require
    [babashka.fs :as fs]
+   [clojure.core.memoize :as memoize]
    [clojure.java.shell :as shell]
-   [clojure.string :as string]))
+   [clojure.string :as string]
+   [eca.shared :as shared]))
 
 (set! *warn-on-reflection* true)
 
+(def ^:private ttl-git-ls-files-ms 5000)
+
+(defn ^:private git-ls-files* [root-path]
+  (try
+    (some-> (shell/sh "git" "ls-files" "--others" "--exclude-standard" "--cached"
+                      :dir root-path)
+            :out
+            (string/split #"\n"))
+
+    (catch Exception _ nil)))
+
+(def ^:private git-ls-files (memoize/ttl git-ls-files* :ttl/threshold ttl-git-ls-files-ms))
+
 (defn filter-allowed [file-paths root-filename config]
   (reduce
    (fn [files {:keys [type]}]
      (case type
-       :gitignore (let [git-files (try (some->> (some-> (shell/sh "git" "ls-files"
-                                                                  :dir root-filename)
-                                                        :out
-                                                        (string/split #"\n"))
-                                                (mapv (comp str fs/canonicalize #(fs/file root-filename %)))
-                                                set)
-                                       (catch Exception _ nil))]
-                    (println git-files (str (last files)))
+       :gitignore (let [git-files (some->> (git-ls-files root-filename)
+                                           (mapv (comp str fs/canonicalize #(fs/file root-filename %)))
+                                           set)]
                     (if (seq git-files)
                       (filter (fn [file]
                                 (contains? git-files (str file)))
@@ -26,3 +36,45 @@
        files))
    file-paths
    (get-in config [:index :ignoreFiles])))
+
+(defn insert-path [tree parts]
+  (if (empty? parts)
+    tree
+    (let [head (first parts)
+          tail (rest parts)]
+      (update tree head #(insert-path (or % {}) tail)))))
+
+(defn tree->str
+  ([tree] (tree->str tree 0))
+  ([tree indent]
+   (let [indent-str (fn [level] (apply str (repeat (* 1 level) " ")))]
+     (apply str
+            (mapcat (fn [[k v]]
+                      (let [current-line (str (indent-str indent) k "\n")
+                            children-str (when (seq v)
+                                           (tree->str v (inc indent)))]
+                        [current-line children-str]))
+                    (sort tree))))))
+
+(defn repo-map [db & {:keys [as-string?]}]
+  (let [tree (reduce
+              (fn [t {:keys [uri]}]
+                (let [root-filename (shared/uri->filename uri)
+                      files (git-ls-files root-filename)]
+                  (merge t
+                         {root-filename
+                          (reduce
+                           (fn [tree path]
+                             (insert-path tree (clojure.string/split path #"/")))
+                           {}
+                           files)})))
+              {}
+              (:workspace-folders db))]
+    (if as-string?
+      (tree->str tree)
+      tree)))
+
+(comment
+  (require 'user)
+  (user/with-workspace-root "file:///home/greg/dev/eca"
+    (println (repo-map user/*db* {:as-string? true}))))

src/eca/features/index.clj

@@ -3,15 +3,16 @@
    [babashka.fs :as fs]
    [clojure.java.shell :as shell]
    [clojure.test :refer [deftest is testing]]
-   [eca.features.index :refer [filter-allowed]]
+   [eca.features.index :as f.index]
+   [eca.test-helper :as h]
    [matcher-combinators.test :refer [match?]]))
 
 (def gitignore-config
   {:index {:ignoreFiles [{:type :gitignore}]}})
 
 (deftest ignore?-test
   (testing "gitignore type"
-    (let [root  "/fake/repo"
+    (let [root "/fake/repo"
           file1 (fs/path root "ignored.txt")
           file2 (fs/path root "not-ignored.txt")]
       (testing "returns filtered files when `git ls-files` works"
@@ -20,11 +21,31 @@
           (is
            (match?
             [file2]
-            (filter-allowed [file1 file2] root gitignore-config)))))
+            (f.index/filter-allowed [file1 file2] root gitignore-config)))))
 
       (testing "returns all files when `git ls-files` exits non-zero"
         (with-redefs [shell/sh (fn [& _args] {:exit 1})]
           (is
            (match?
-            [file1 file2]
-            (filter-allowed [file1 file2] root gitignore-config))))))))
+            [file2]
+            (f.index/filter-allowed [file1 file2] root gitignore-config))))))))
+
+(deftest repo-map-test
+  (testing "returns correct tree for a simple git repo"
+    (with-redefs [f.index/git-ls-files (constantly ["README.md"
+                                                    "src/eca/core.clj"
+                                                    "test/eca/core_test.clj"])]
+      (is (match?
+           {"/fake/repo"
+            {"README.md" {}
+             "src" {"eca" {"core.clj" {}}}
+             "test" {"eca" {"core_test.clj" {}}}}}
+           (eca.features.index/repo-map {:workspace-folders [{:uri (h/file-uri "file:///fake/repo")}]})))))
+  (testing "returns string tree with as-string? true"
+    (with-redefs [f.index/git-ls-files (constantly ["foo.clj" "bar/baz.clj"])]
+      (is (= (str "/fake/repo\n"
+                  " bar\n"
+                  "  baz.clj\n"
+                  " foo.clj\n")
+             (eca.features.index/repo-map {:workspace-folders [{:uri (h/file-uri "file:///fake/repo")}]}
+                                          {:as-string? true}))))))