perf: Decouple filesystem walk from git index construction

Split find_untracked_files into two phases that can run in parallel:

1. walk_candidate_files (I/O-bound): Enumerates all non-gitignored files
   within scope using the ignore crate's native gitignore support. Only
   needs the git root path and package prefixes — no tracked index.

2. filter_untracked_from_candidates (CPU-bound): Binary-searches each
   candidate against ls_tree_hashes and status_entries to identify truly
   untracked files. Runs after the tracked index is ready.

The git root is sent via a oneshot channel as soon as SCM::new() resolves
it (~5ms), while new_from_gix_index continues (~267ms). The walk starts
immediately and runs in parallel with index construction.

Benchmark (110-package monorepo, 30 runs, sandbox):
  baseline: 853ms ± 19ms
  improved: 619ms ± 6ms  (1.38x faster)

Author: anthonyshew

crates/turborepo-lib/src/run/builder.rs

@@ -262,7 +262,9 @@ impl RunBuilder {
         let start_at = Local::now();
 
         let (tracked_index_tx, tracked_index_rx) =
-            tokio::sync::oneshot::channel::<(SCM, Option<turborepo_scm::RepoGitIndex>)>();
+            tokio::sync::oneshot::channel::<Option<turborepo_scm::RepoGitIndex>>();
+        let (git_root_tx, git_root_rx) =
+            tokio::sync::oneshot::channel::<Option<turbopath::AbsoluteSystemPathBuf>>();
         let scm_task = {
             let repo_root = self.repo_root.clone();
             let git_root = self.opts.git_root.clone();
@@ -271,8 +273,11 @@ impl RunBuilder {
                     Some(root) => SCM::new_with_git_root(&repo_root, root),
                     None => SCM::new(&repo_root),
                 };
+                // Send git root immediately so the filesystem walk can start
+                // while index construction continues.
+                let _ = git_root_tx.send(scm.git_root().map(|r| r.to_owned()));
                 let repo_index = scm.build_tracked_repo_index_eager();
-                let _ = tracked_index_tx.send((scm.clone(), repo_index));
+                let _ = tracked_index_tx.send(repo_index);
                 scm
             })
         };
@@ -349,40 +354,39 @@ impl RunBuilder {
         repo_telemetry.track_size(pkg_dep_graph.len());
         run_telemetry.track_run_type(self.opts.run_opts.dry_run.is_some());
 
-        // Spawn the untracked-file walk as soon as the package graph is ready.
-        // We use all-package prefixes (superset of any filtered selection) so
-        // the walk can start before filter resolution. Per-package hash queries
-        // use binary-search range scoping, so extra untracked files outside a
-        // queried package are never returned.
+        // Spawn the filesystem walk as soon as the git root is resolved.
+        // It only needs the git root and package prefixes, not the tracked
+        // index. The walk runs in parallel with new_from_gix_index (~267ms).
         let all_prefixes = Self::all_package_prefixes(&pkg_dep_graph);
-        let repo_index_task = if all_prefixes.is_empty() {
+        let walk_task = if all_prefixes.is_empty() {
             None
         } else {
             Some(tokio::task::spawn(async move {
-                let (scm, tracked_index) = match tracked_index_rx.await {
-                    Ok(pair) => pair,
-                    Err(_) => return None,
+                let git_root = match git_root_rx.await {
+                    Ok(Some(root)) => root,
+                    _ => return None,
                 };
-                let tracked_index = tracked_index?;
                 tokio::task::spawn_blocking(move || {
-                    let _span = tracing::info_span!("repo_index_scope_untracked").entered();
-                    let mut repo_index = tracked_index;
-                    match scm.populate_repo_index_untracked(&mut repo_index, &all_prefixes) {
-                        Ok(()) => Some(repo_index),
-                        Err(err) => {
-                            tracing::debug!(
-                                "failed to scope repo git index with untracked files: {}. Will \
-                                 hash per-package.",
-                                err,
-                            );
-                            None
-                        }
-                    }
+                    let _span = tracing::info_span!("walk_candidate_files").entered();
+                    turborepo_scm::walk_candidate_files(git_root.as_std_path(), Some(&all_prefixes))
+                        .ok()
                 })
                 .await
                 .ok()?
             }))
         };
+
+        // Combine the walk results with the tracked index once both are ready.
+        let repo_index_task = walk_task.map(|walk_task| {
+            tokio::task::spawn(async move {
+                let (candidates, tracked_index) = tokio::join!(walk_task, tracked_index_rx);
+                let candidates = candidates.ok()??;
+                let tracked_index = tracked_index.ok()??;
+                let mut repo_index = tracked_index;
+                repo_index.populate_untracked_from_candidates(candidates);
+                Some(repo_index)
+            })
+        });
         let micro_frontend_configs = {
             let _span = tracing::info_span!("micro_frontends_from_disk").entered();
             match MicrofrontendsConfigs::from_disk(&self.repo_root, &pkg_dep_graph) {

crates/turborepo-scm/src/git_index_regression_tests.rs

@@ -12,7 +12,7 @@
 
 use turbopath::{AnchoredSystemPathBuf, RelativeUnixPathBuf};
 
-use crate::{GitHashes, RepoGitIndex, SCM, test_utils};
+use crate::{GitHashes, RepoGitIndex, SCM, test_utils, walk_candidate_files};
 
 fn path(s: &str) -> RelativeUnixPathBuf {
     RelativeUnixPathBuf::new(s).unwrap()
@@ -61,6 +61,23 @@ impl TestRepo {
             .expect("failed to build repo index")
     }
 
+    fn build_split_repo_index(&self, prefixes: &[&str]) -> RepoGitIndex {
+        let scm = self.scm();
+        let prefix_paths = prefixes
+            .iter()
+            .map(|prefix| path(prefix))
+            .collect::<Vec<_>>();
+
+        let candidates = walk_candidate_files(self.root.as_std_path(), Some(&prefix_paths))
+            .expect("walk candidates failed");
+
+        let mut index = scm
+            .build_tracked_repo_index_eager()
+            .expect("failed to build tracked repo index");
+        index.populate_untracked_from_candidates(candidates);
+        index
+    }
+
     fn build_scoped_repo_index(&self, prefixes: &[&str]) -> RepoGitIndex {
         let scm = self.scm();
         let mut index = scm
@@ -1421,6 +1438,157 @@ fn test_superset_walk_untracked_in_other_pkg_does_not_affect_queried_pkg() {
     assert!(pkg_b_hashes.contains_key(&path("extra5.ts")));
 }
 
+// Category 6: Split walk/filter regression tests
+//
+// These tests validate that the two-phase approach (walk_candidate_files
+// followed by populate_untracked_from_candidates) produces identical
+// results to the original single-pass find_untracked_files.
+
+#[test]
+fn test_split_walk_matches_original_path() {
+    let repo = TestRepo::new();
+
+    repo.create_file("pkg-a/src/index.ts", "a code");
+    repo.create_file("pkg-a/package.json", "{}");
+    repo.create_file("pkg-b/src/index.ts", "b code");
+    repo.create_file("pkg-b/package.json", "{}");
+    repo.create_file("package.json", "{}");
+    repo.commit_all();
+
+    repo.create_file("pkg-a/untracked.ts", "new a");
+    repo.create_file("pkg-b/untracked.ts", "new b");
+
+    let original = repo.build_scoped_repo_index(&["pkg-a", "pkg-b"]);
+    let split = repo.build_split_repo_index(&["pkg-a", "pkg-b"]);
+
+    let orig_a = repo.get_hashes_with_index("pkg-a", &original);
+    let split_a = repo.get_hashes_with_index("pkg-a", &split);
+    assert_eq!(orig_a, split_a, "pkg-a: split walk must match original");
+
+    let orig_b = repo.get_hashes_with_index("pkg-b", &original);
+    let split_b = repo.get_hashes_with_index("pkg-b", &split);
+    assert_eq!(orig_b, split_b, "pkg-b: split walk must match original");
+
+    let no_idx_a = repo.get_hashes_no_index("pkg-a");
+    let no_idx_b = repo.get_hashes_no_index("pkg-b");
+    assert_eq!(split_a, no_idx_a, "pkg-a: split vs no-index");
+    assert_eq!(split_b, no_idx_b, "pkg-b: split vs no-index");
+}
+
+#[test]
+fn test_split_walk_respects_gitignore() {
+    let repo = TestRepo::new();
+
+    repo.create_gitignore(".gitignore", "*.log\nbuild/\nnode_modules/\n");
+    repo.create_gitignore("pkg-b/.gitignore", "tmp/\n");
+    repo.create_file("pkg-a/src/index.ts", "a");
+    repo.create_file("pkg-a/package.json", "{}");
+    repo.create_file("pkg-b/src/index.ts", "b");
+    repo.create_file("pkg-b/package.json", "{}");
+    repo.create_file("package.json", "{}");
+    repo.commit_all();
+
+    repo.create_file("pkg-a/debug.log", "log");
+    repo.create_file("pkg-a/build/out.js", "out");
+    repo.create_file("pkg-a/node_modules/dep/index.js", "dep");
+    repo.create_file("pkg-b/tmp/cache.dat", "cache");
+    repo.create_file("pkg-b/build/out.js", "out");
+    repo.create_file("pkg-a/new.ts", "new");
+    repo.create_file("pkg-b/new.ts", "new");
+
+    let original = repo.build_scoped_repo_index(&["pkg-a", "pkg-b"]);
+    let split = repo.build_split_repo_index(&["pkg-a", "pkg-b"]);
+
+    let orig_a = repo.get_hashes_with_index("pkg-a", &original);
+    let split_a = repo.get_hashes_with_index("pkg-a", &split);
+    assert_eq!(
+        orig_a, split_a,
+        "pkg-a: split must match original with gitignore"
+    );
+    assert!(split_a.contains_key(&path("new.ts")));
+    assert!(!split_a.contains_key(&path("debug.log")));
+    assert!(!split_a.contains_key(&path("build/out.js")));
+    assert!(!split_a.contains_key(&path("node_modules/dep/index.js")));
+
+    let orig_b = repo.get_hashes_with_index("pkg-b", &original);
+    let split_b = repo.get_hashes_with_index("pkg-b", &split);
+    assert_eq!(
+        orig_b, split_b,
+        "pkg-b: split must match original with gitignore"
+    );
+    assert!(split_b.contains_key(&path("new.ts")));
+    assert!(!split_b.contains_key(&path("tmp/cache.dat")));
+}
+
+#[test]
+fn test_split_walk_with_untracked_gitignore() {
+    let repo = TestRepo::new();
+
+    repo.create_file("pkg-a/src/index.ts", "a");
+    repo.create_file("pkg-a/package.json", "{}");
+    repo.create_file("package.json", "{}");
+    repo.commit_all();
+
+    repo.create_gitignore("pkg-a/.gitignore", "generated/\n");
+    repo.create_file("pkg-a/generated/output.js", "out");
+    repo.create_file("pkg-a/new.ts", "new");
+
+    let original = repo.build_scoped_repo_index(&["pkg-a"]);
+    let split = repo.build_split_repo_index(&["pkg-a"]);
+
+    let orig_a = repo.get_hashes_with_index("pkg-a", &original);
+    let split_a = repo.get_hashes_with_index("pkg-a", &split);
+
+    assert!(split_a.contains_key(&path("new.ts")));
+    assert!(orig_a.contains_key(&path("new.ts")));
+    assert!(
+        !split_a.contains_key(&path("generated/output.js")),
+        "split walk should respect untracked .gitignore"
+    );
+    assert!(
+        !orig_a.contains_key(&path("generated/output.js")),
+        "original walk should respect untracked .gitignore"
+    );
+    assert!(split_a.contains_key(&path(".gitignore")));
+    assert!(orig_a.contains_key(&path(".gitignore")));
+}
+
+#[test]
+fn test_split_walk_nested_gitignore_scoping() {
+    let repo = TestRepo::new();
+
+    repo.create_gitignore(".gitignore", "*.log\n");
+    repo.create_gitignore("pkg-a/.gitignore", "output/\n");
+    repo.create_file("pkg-a/src/index.ts", "a");
+    repo.create_file("pkg-a/package.json", "{}");
+    repo.create_file("pkg-b/src/index.ts", "b");
+    repo.create_file("pkg-b/package.json", "{}");
+    repo.create_file("package.json", "{}");
+    repo.commit_all();
+
+    repo.create_file("pkg-a/output/bundle.js", "a bundle");
+    repo.create_file("pkg-b/output/bundle.js", "b bundle");
+
+    let split = repo.build_split_repo_index(&["pkg-a", "pkg-b"]);
+
+    let split_a = repo.get_hashes_with_index("pkg-a", &split);
+    assert!(
+        !split_a.contains_key(&path("output/bundle.js")),
+        "pkg-a output/ should be ignored by pkg-a/.gitignore"
+    );
+
+    let split_b = repo.get_hashes_with_index("pkg-b", &split);
+    assert!(
+        split_b.contains_key(&path("output/bundle.js")),
+        "pkg-b output/ should NOT be ignored — the output/ rule is scoped to pkg-a"
+    );
+
+    let no_idx_a = repo.get_hashes_no_index("pkg-a");
+    let no_idx_b = repo.get_hashes_no_index("pkg-b");
+    assert_eq!(split_a, no_idx_a, "pkg-a split vs no-index");
+    assert_eq!(split_b, no_idx_b, "pkg-b split vs no-index");
+}
+
 #[test]
 fn test_nested_gitignore_scoping() {
     // Gitignore rules in a nested .gitignore should only apply to that

crates/turborepo-scm/src/lib.rs

@@ -34,7 +34,7 @@ mod git_index_regression_tests;
 #[cfg(test)]
 mod test_utils;
 
-pub use repo_index::RepoGitIndex;
+pub use repo_index::{RepoGitIndex, walk_candidate_files};
 pub use turborepo_hash::OidHash;
 pub use worktree::WorktreeInfo;
 
@@ -311,6 +311,13 @@ impl SCM {
         matches!(self, SCM::Manual)
     }
 
+    pub fn git_root(&self) -> Option<&AbsoluteSystemPath> {
+        match self {
+            SCM::Git(git) => Some(&git.root),
+            SCM::Manual => None,
+        }
+    }
+
     /// Build a repo-wide git index that caches `git ls-tree` and `git status`
     /// results. Returns `None` for manual SCM mode or when the package count
     /// is too small to benefit. Callers should build this once before parallel