Improve context query performance

Author: ericdallo

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+- Improve context query performance.
+
 ## 0.4.2
 
 - Fix output of errored tool calls.

src/eca/features/chat.clj

@@ -253,6 +253,11 @@
      :model chosen-model
      :status :success}))
 
+(defn ^:private contexts-for [root-filename query config]
+  (let [all-files (fs/glob root-filename (str "**" (or query "") "**"))
+        allowed-files (f.index/filter-allowed all-files root-filename config)]
+    allowed-files))
+
 (defn query-context
   [{:keys [query contexts chat-id]}
    db*
@@ -261,14 +266,7 @@
                                     (comp
                                      (map :uri)
                                      (map shared/uri->filename)
-                                     (mapcat (fn [root-filename]
-                                               (let [all-files (fs/glob root-filename (str "**" (or query "") "**"))
-                                                     all-dirs (filter fs/directory? all-files)
-                                                     excluded-dirs (filter #(f.index/ignore? (str %) root-filename config) all-dirs)]
-                                                 (->> all-files
-                                                      (remove (fn [path]
-                                                                (or (some #(fs/starts-with? (str path) %) excluded-dirs)
-                                                                    (f.index/ignore? (str path) root-filename config))))))))
+                                     (mapcat #(contexts-for % query config))
                                      (take 200) ;; for performance, user can always make query specific for better results.
                                      (map (fn [file-or-dir]
                                             {:type (if (fs/directory? file-or-dir)

src/eca/features/index.clj

@@ -1,17 +1,28 @@
 (ns eca.features.index
   (:require
-   [clojure.java.shell :as shell]))
+   [babashka.fs :as fs]
+   [clojure.java.shell :as shell]
+   [clojure.string :as string]))
 
 (set! *warn-on-reflection* true)
 
-(defn ignore? [filename root-filename config]
-  (boolean
-   (some
-    (fn [{:keys [type]}]
-      (case type
-        :gitignore (= 0 (:exit
-                         (try (shell/sh "git" "check-ignore" filename "--quiet"
-                                        :dir root-filename)
-                              (catch Exception _ nil))))
-        nil))
-    (get-in config [:index :ignoreFiles]))))
+(defn filter-allowed [file-paths root-filename config]
+  (reduce
+   (fn [files {:keys [type]}]
+     (case type
+       :gitignore (let [git-files (try (some->> (some-> (shell/sh "git" "ls-files"
+                                                                  :dir root-filename)
+                                                        :out
+                                                        (string/split #"\n"))
+                                                (mapv (comp str fs/canonicalize #(fs/file root-filename %)))
+                                                set)
+                                       (catch Exception _ nil))]
+                    (println git-files (str (last files)))
+                    (if (seq git-files)
+                      (filter (fn [file]
+                                (contains? git-files (str file)))
+                              files)
+                      files))
+       files))
+   file-paths
+   (get-in config [:index :ignoreFiles])))

test/eca/features/index_test.clj

@@ -1,28 +1,30 @@
 (ns eca.features.index-test
   (:require
+   [babashka.fs :as fs]
    [clojure.java.shell :as shell]
    [clojure.test :refer [deftest is testing]]
-   [eca.features.index :refer [ignore?]]))
+   [eca.features.index :refer [filter-allowed]]
+   [matcher-combinators.test :refer [match?]]))
 
-;; a helper config that only looks at :gitignore entries
 (def gitignore-config
   {:index {:ignoreFiles [{:type :gitignore}]}})
 
 (deftest ignore?-test
   (testing "gitignore type"
     (let [root  "/fake/repo"
-          file1 "ignored.txt"
-          file2 "not-ignored.txt"]
-      (testing "returns true when `git check-ignore` exits 0"
-        (with-redefs [shell/sh (fn [& _args] {:exit 0})]
-          (is (true?  (ignore? file1 root gitignore-config)))
-          ;; even a different filename still is seen as ignored
-          (is (true?  (ignore? file2 root gitignore-config)))))
+          file1 (fs/path root "ignored.txt")
+          file2 (fs/path root "not-ignored.txt")]
+      (testing "returns filtered files when `git ls-files` works"
+        (with-redefs [shell/sh (fn [& _args] {:exit 0 :out "not-ignored.txt"})
+                      fs/canonicalize #(fs/path root %)]
+          (is
+           (match?
+            [file2]
+            (filter-allowed [file1 file2] root gitignore-config)))))
 
-      (testing "returns false when `git check-ignore` exits non-zero"
-        (with-redefs [shell/sh (fn [& _] {:exit 1})]
-          (is (false? (ignore? file1 root gitignore-config)))))
-
-      (testing "returns false when `git check-ignore` throws an exception"
-        (with-redefs [shell/sh (fn [& _] (throw (Exception. "boom")))]
-          (is (false? (ignore? file1 root gitignore-config))))))))
+      (testing "returns all files when `git ls-files` exits non-zero"
+        (with-redefs [shell/sh (fn [& _args] {:exit 1})]
+          (is
+           (match?
+            [file1 file2]
+            (filter-allowed [file1 file2] root gitignore-config))))))))