parallel walk dir - 25% faster

Author: perryqh

src/cli.rs

@@ -27,7 +27,7 @@ enum Command {
         visible_alias = "g"
     )]
     Generate {
-        #[arg(long, short,default_value = "false", help = "Skip staging the CODEOWNERS file")]
+        #[arg(long, short, default_value = "false", help = "Skip staging the CODEOWNERS file")]
         skip_stage: bool,
     },
 
@@ -39,7 +39,7 @@ enum Command {
 
     #[clap(about = "Chains both `generate` and `validate` commands.", visible_alias = "gv")]
     GenerateAndValidate {
-        #[arg(long, short,default_value = "false", help = "Skip staging the CODEOWNERS file")]
+        #[arg(long, short, default_value = "false", help = "Skip staging the CODEOWNERS file")]
         skip_stage: bool,
     },
 

src/project_builder.rs

@@ -1,11 +1,12 @@
 use std::{
     fs::File,
     path::{Path, PathBuf},
+    sync::{Arc, Mutex},
 };
 
 use error_stack::{Result, ResultExt};
 use fast_glob::glob_match;
-use ignore::WalkBuilder;
+use ignore::{WalkBuilder, WalkParallel, WalkState};
 use rayon::iter::{IntoParallelIterator, ParallelIterator};
 use tracing::{instrument, warn};
 
@@ -53,12 +54,29 @@ impl<'a> ProjectBuilder<'a> {
         let mut entry_types = Vec::with_capacity(INITIAL_VECTOR_CAPACITY);
         let mut builder = WalkBuilder::new(&self.base_path);
         builder.hidden(false);
-        let walkdir = builder.build();
+        let walk_parallel: WalkParallel = builder.build_parallel();
 
-        for entry in walkdir {
-            let entry = entry.change_context(Error::Io)?;
+        let collected = Arc::new(Mutex::new(Vec::with_capacity(INITIAL_VECTOR_CAPACITY)));
+        let collected_for_threads = Arc::clone(&collected);
+
+        walk_parallel.run(move || {
+            let collected = Arc::clone(&collected_for_threads);
+            Box::new(move |res| {
+                if let Ok(entry) = res {
+                    if let Ok(mut v) = collected.lock() {
+                        v.push(entry);
+                    }
+                }
+                WalkState::Continue
+            })
+        });
+
+        // Process sequentially with &mut self
+        let collected_entries = Arc::try_unwrap(collected).unwrap().into_inner().unwrap();
+        for entry in collected_entries {
             entry_types.push(self.build_entry_type(entry)?);
         }
+
         self.build_project_from_entry_types(entry_types)
     }
 

src/runner.rs

@@ -244,11 +244,9 @@ impl Runner {
         })
     }
 
-    pub fn validate(&self ) -> RunResult {
+    pub fn validate(&self) -> RunResult {
         match self.ownership.validate() {
-            Ok(_) => {
-                RunResult::default()
-            },
+            Ok(_) => RunResult::default(),
             Err(err) => RunResult {
                 validation_errors: vec![format!("{}", err)],
                 ..Default::default()
@@ -267,7 +265,7 @@ impl Runner {
                     self.git_stage();
                 }
                 RunResult::default()
-            },
+            }
             Err(err) => RunResult {
                 io_errors: vec![err.to_string()],
                 ..Default::default()

tests/git_stage_test.rs

@@ -30,11 +30,25 @@ fn init_git_repo(path: &Path) {
         .current_dir(path)
         .output()
         .expect("failed to run git init");
-    assert!(status.status.success(), "git init failed: {}", String::from_utf8_lossy(&status.stderr));
+    assert!(
+        status.status.success(),
+        "git init failed: {}",
+        String::from_utf8_lossy(&status.stderr)
+    );
 
     // Configure a dummy identity to appease git if commits ever happen; not strictly needed for staging
-    let _ = Command::new("git").arg("config").arg("user.email").arg("[email protected]").current_dir(path).output();
-    let _ = Command::new("git").arg("config").arg("user.name").arg("Test User").current_dir(path).output();
+    let _ = Command::new("git")
+        .arg("config")
+        .arg("user.email")
+        .arg("[email protected]")
+        .current_dir(path)
+        .output();
+    let _ = Command::new("git")
+        .arg("config")
+        .arg("user.name")
+        .arg("Test User")
+        .current_dir(path)
+        .output();
 }
 
 fn is_file_staged(repo_root: &Path, rel_path: &str) -> bool {
@@ -46,7 +60,11 @@ fn is_file_staged(repo_root: &Path, rel_path: &str) -> bool {
         .current_dir(repo_root)
         .output()
         .expect("failed to run git diff --cached");
-    assert!(output.status.success(), "git diff failed: {}", String::from_utf8_lossy(&output.stderr));
+    assert!(
+        output.status.success(),
+        "git diff failed: {}",
+        String::from_utf8_lossy(&output.stderr)
+    );
     let stdout = String::from_utf8_lossy(&output.stdout);
     stdout.lines().any(|line| line.trim() == rel_path)
 }
@@ -77,7 +95,11 @@ fn test_generate_stages_codeowners() {
     let run_config = build_run_config(&temp_root, ".github/CODEOWNERS");
     let result = runner::generate(&run_config, true);
     assert!(result.io_errors.is_empty(), "io_errors: {:?}", result.io_errors);
-    assert!(result.validation_errors.is_empty(), "validation_errors: {:?}", result.validation_errors);
+    assert!(
+        result.validation_errors.is_empty(),
+        "validation_errors: {:?}",
+        result.validation_errors
+    );
 
     // Assert CODEOWNERS file exists and is staged
     let rel_path = ".github/CODEOWNERS";
@@ -99,12 +121,14 @@ fn test_generate_and_validate_stages_codeowners() {
     let run_config = build_run_config(&temp_root, ".github/CODEOWNERS");
     let result = runner::generate_and_validate(&run_config, vec![], true);
     assert!(result.io_errors.is_empty(), "io_errors: {:?}", result.io_errors);
-    assert!(result.validation_errors.is_empty(), "validation_errors: {:?}", result.validation_errors);
+    assert!(
+        result.validation_errors.is_empty(),
+        "validation_errors: {:?}",
+        result.validation_errors
+    );
 
     // Assert CODEOWNERS file exists and is staged
     let rel_path = ".github/CODEOWNERS";
     assert!(run_config.codeowners_file_path.exists(), "CODEOWNERS file was not created");
     assert!(is_file_staged(&run_config.project_root, rel_path), "CODEOWNERS file was not staged");
 }
-
-