fix/feat: Improve @ file tagging, reduce lag (#241)

Author: charlesvien

apps/array/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@posthog/array",
-  "version": "0.12.0",
+  "version": "0.13.0",
   "description": "Array - PostHog desktop task manager",
   "main": ".vite/build/index.js",
   "versionHash": "dynamic",

apps/array/src/main/preload.ts

@@ -147,8 +147,9 @@ contextBridge.exposeInMainWorld("electronAPI", {
   listRepoFiles: (
     repoPath: string,
     query?: string,
+    limit?: number,
   ): Promise<Array<{ path: string; name: string }>> =>
-    ipcRenderer.invoke("list-repo-files", repoPath, query),
+    ipcRenderer.invoke("list-repo-files", repoPath, query, limit),
   clearRepoFileCache: (repoPath: string): Promise<void> =>
     ipcRenderer.invoke("clear-repo-file-cache", repoPath),
   agentStart: async (

apps/array/src/main/services/fs.ts

@@ -24,77 +24,88 @@ const repoFileCache = new Map<
 >();
 const CACHE_TTL = 30000; // 30 seconds
 
-async function getGitIgnoredFiles(repoPath: string): Promise<Set<string>> {
+/**
+ * List files using git ls-files - fast and works for any git repository.
+ * This is much faster than recursive fs.readdir for large codebases.
+ */
+async function listFilesWithGit(
+  repoPath: string,
+  changedFiles: Set<string>,
+): Promise<FileEntry[]> {
   try {
-    const { stdout } = await execAsync(
-      "git ls-files --others --ignored --exclude-standard",
-      { cwd: repoPath },
+    const { stdout: trackedStdout } = await execAsync("git ls-files", {
+      cwd: repoPath,
+      maxBuffer: 50 * 1024 * 1024,
+    });
+
+    const { stdout: untrackedStdout } = await execAsync(
+      "git ls-files --others --exclude-standard",
+      { cwd: repoPath, maxBuffer: 50 * 1024 * 1024 },
     );
-    return new Set(
-      stdout
-        .split("\n")
-        .filter(Boolean)
-        .map((f) => path.join(repoPath, f)),
-    );
-  } catch {
-    // If git command fails, return empty set
-    return new Set();
+
+    const allFiles = [
+      ...trackedStdout.split("\n").filter(Boolean),
+      ...untrackedStdout.split("\n").filter(Boolean),
+    ];
+
+    return allFiles.map((relativePath) => ({
+      path: relativePath,
+      name: path.basename(relativePath),
+      changed: changedFiles.has(relativePath),
+    }));
+  } catch (error) {
+    log.error("Error listing files with git:", error);
+    return [];
   }
 }
 
-async function listFilesRecursive(
-  dirPath: string,
-  ignoredFiles: Set<string>,
-  baseDir: string,
+/**
+ * List files with early termination using grep and head.
+ * Returns limited results directly from git without loading all files into memory.
+ */
+async function listFilesWithQuery(
+  repoPath: string,
+  query: string,
+  limit: number,
   changedFiles: Set<string>,
 ): Promise<FileEntry[]> {
-  const files: FileEntry[] = [];
-
   try {
-    const entries = await fsPromises.readdir(dirPath, { withFileTypes: true });
-
-    for (const entry of entries) {
-      const fullPath = path.join(dirPath, entry.name);
-      const relativePath = path.relative(baseDir, fullPath);
-
-      // Skip .git directory, node_modules, and common build dirs
-      if (
-        entry.name === ".git" ||
-        entry.name === "node_modules" ||
-        entry.name === "dist" ||
-        entry.name === "build" ||
-        entry.name === "__pycache__"
-      ) {
-        continue;
-      }
-
-      // Skip git-ignored files
-      if (ignoredFiles.has(fullPath)) {
-        continue;
-      }
-
-      if (entry.isDirectory()) {
-        const subFiles = await listFilesRecursive(
-          fullPath,
-          ignoredFiles,
-          baseDir,
-          changedFiles,
-        );
-        files.push(...subFiles);
-      } else if (entry.isFile()) {
-        files.push({
-          path: relativePath,
-          name: entry.name,
-          changed: changedFiles.has(relativePath),
-        });
-      }
-    }
+    // escape special regex characters in the query for grep
+    const escapedQuery = query.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+
+    // grep -i for case-insensitive matching, head for early termination
+    const grepCmd = `grep -i "${escapedQuery}" | head -n ${limit}`;
+
+    // the || true prevents error when grep finds no matches
+    const [trackedResult, untrackedResult] = await Promise.all([
+      execAsync(`git ls-files | ${grepCmd} || true`, {
+        cwd: repoPath,
+        maxBuffer: 1024 * 1024,
+      }),
+      execAsync(
+        `git ls-files --others --exclude-standard | ${grepCmd} || true`,
+        {
+          cwd: repoPath,
+          maxBuffer: 1024 * 1024,
+        },
+      ),
+    ]);
+
+    const trackedFiles = trackedResult.stdout.split("\n").filter(Boolean);
+    const untrackedFiles = untrackedResult.stdout.split("\n").filter(Boolean);
+
+    // combine and limit to requested amount (in case both sources have results)
+    const allFiles = [...trackedFiles, ...untrackedFiles].slice(0, limit);
+
+    return allFiles.map((relativePath) => ({
+      path: relativePath,
+      name: path.basename(relativePath),
+      changed: changedFiles.has(relativePath),
+    }));
   } catch (error) {
-    // Skip directories we can't read
-    log.error(`Error reading directory ${dirPath}:`, error);
+    log.error("Error listing files with query:", error);
+    return [];
   }
-
-  return files;
 }
 
 export function registerFsIpc(): void {
@@ -104,11 +115,26 @@ export function registerFsIpc(): void {
       _event: IpcMainInvokeEvent,
       repoPath: string,
       query?: string,
+      limit?: number,
     ): Promise<FileEntry[]> => {
       if (!repoPath) return [];
 
+      const resultLimit = limit ?? 50;
+
       try {
-        // Check cache
+        const changedFiles = await getChangedFilesForRepo(repoPath);
+
+        // when there is a query, use early termination with grep + head
+        // this avoids loading all files into memory for filtered searches
+        if (query?.trim()) {
+          return await listFilesWithQuery(
+            repoPath,
+            query.trim(),
+            resultLimit,
+            changedFiles,
+          );
+        }
+
         const cached = repoFileCache.get(repoPath);
         const now = Date.now();
 
@@ -117,40 +143,15 @@ export function registerFsIpc(): void {
         if (cached && now - cached.timestamp < CACHE_TTL) {
           allFiles = cached.files;
         } else {
-          // Get git-ignored files
-          const ignoredFiles = await getGitIgnoredFiles(repoPath);
-
-          // Get changed files from git
-          const changedFiles = await getChangedFilesForRepo(repoPath);
+          allFiles = await listFilesWithGit(repoPath, changedFiles);
 
-          // List all files
-          allFiles = await listFilesRecursive(
-            repoPath,
-            ignoredFiles,
-            repoPath,
-            changedFiles,
-          );
-
-          // Update cache
           repoFileCache.set(repoPath, {
             files: allFiles,
             timestamp: now,
           });
         }
 
-        // Filter by query if provided
-        if (query?.trim()) {
-          const lowerQuery = query.toLowerCase();
-          return allFiles
-            .filter(
-              (f) =>
-                f.path.toLowerCase().includes(lowerQuery) ||
-                f.name.toLowerCase().includes(lowerQuery),
-            )
-            .slice(0, 50); // Limit search results
-        }
-
-        return allFiles; // Return all files for full tree view
+        return allFiles.slice(0, resultLimit);
       } catch (error) {
         log.error("Error listing repo files:", error);
         return [];

apps/array/src/renderer/features/sessions/components/MessageEditor.tsx

@@ -231,12 +231,20 @@ export const MessageEditor = forwardRef<
       }
     };
     const [mentionItems, setMentionItems] = useState<MentionItem[]>([]);
-    const mentionItemsRef = useRef(mentionItems);
     const repoPathRef = useRef(repoPath);
     const onSubmitRef = useRef(onSubmit);
+    const componentRef = useRef<ReactRenderer<MentionListRef> | null>(null);
+    const commandRef = useRef<
+      ((item: { id: string; label: string; type?: string }) => void) | null
+    >(null);
 
     useEffect(() => {
-      mentionItemsRef.current = mentionItems;
+      if (componentRef.current && commandRef.current) {
+        componentRef.current.updateProps({
+          items: mentionItems,
+          command: commandRef.current,
+        });
+      }
     }, [mentionItems]);
 
     useEffect(() => {
@@ -301,10 +309,14 @@ export const MessageEditor = forwardRef<
                 setMentionItems([]);
                 return [];
               }
+
+              setMentionItems([]);
+
               try {
                 const results = await window.electronAPI?.listRepoFiles(
                   repoPathRef.current,
                   query,
+                  10,
                 );
                 const items = (results || []).map((file) => ({
                   path: file.path,
@@ -320,8 +332,6 @@ export const MessageEditor = forwardRef<
               }
             },
             render: () => {
-              let component: ReactRenderer<MentionListRef> | null = null;
-
               const updatePosition = (ed: Editor, element: HTMLElement) => {
                 const refRect = posToDOMRect(
                   ed.view,
@@ -338,36 +348,41 @@ export const MessageEditor = forwardRef<
 
               return {
                 onStart: (props: SuggestionProps) => {
-                  component = new ReactRenderer(MentionList, {
+                  commandRef.current = props.command;
+
+                  const component = new ReactRenderer(MentionList, {
                     props: {
-                      items: mentionItemsRef.current,
+                      items: [],
                       command: props.command,
                     },
                     editor: props.editor,
                   });
+
+                  componentRef.current = component;
+
                   if (!props.clientRect) return;
                   component.element.style.position = "absolute";
                   document.body.appendChild(component.element);
                   updatePosition(props.editor, component.element);
                 },
                 onUpdate: (props: SuggestionProps) => {
-                  component?.updateProps({
-                    items: mentionItemsRef.current,
-                    command: props.command,
-                  });
-                  if (!props.clientRect || !component) return;
-                  updatePosition(props.editor, component.element);
+                  if (!props.clientRect || !componentRef.current) return;
+                  updatePosition(props.editor, componentRef.current.element);
                 },
                 onKeyDown: (props: SuggestionKeyDownProps) => {
                   if (props.event.key === "Escape") {
-                    component?.destroy();
+                    componentRef.current?.destroy();
+                    componentRef.current = null;
+                    commandRef.current = null;
                     return true;
                   }
-                  return component?.ref?.onKeyDown?.(props) ?? false;
+                  return componentRef.current?.ref?.onKeyDown?.(props) ?? false;
                 },
                 onExit: () => {
-                  component?.element.remove();
-                  component?.destroy();
+                  componentRef.current?.element.remove();
+                  componentRef.current?.destroy();
+                  componentRef.current = null;
+                  commandRef.current = null;
                 },
               };
             },