Skip to content

Commit 610fb2c

Browse files
committed
Two-phase approach speeds up cold-cache generate
Phase 1 - Discovery (WalkParallel) - Only collect file paths - no I/O - Create placeholder ProjectFile with owner: None - WalkParallel threads stay fast and unblocked Phase 2 - Processing (Rayon) - After all files discovered, process them in parallel with rayon - Call project_file_builder.build() to read files and check cache - Rayon's work-stealing thread pool is optimized for I/O-heavy tasks - No contention with file discovery
1 parent c3cace6 commit 610fb2c

File tree

1 file changed

+17
-2
lines changed

1 file changed

+17
-2
lines changed

src/project_builder.rs

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -170,8 +170,11 @@ impl<'a> ProjectBuilder<'a> {
170170
Ok(EntryType::TeamFile(absolute_path.to_owned(), relative_path.to_owned()))
171171
}
172172
_ if matches_globs(&relative_path, &self.config.owned_globs) && !matches_globs(&relative_path, &self.config.unowned_globs) => {
173-
let project_file = self.project_file_builder.build(absolute_path.to_path_buf());
174-
Ok(EntryType::OwnedFile(project_file))
173+
// Defer file processing - just collect the path
174+
Ok(EntryType::OwnedFile(ProjectFile {
175+
path: absolute_path.to_path_buf(),
176+
owner: None, // Will be populated later
177+
}))
175178
}
176179
_ => Ok(EntryType::NullEntry()),
177180
}
@@ -273,6 +276,18 @@ impl<'a> ProjectBuilder<'a> {
273276
acc
274277
},
275278
);
279+
// Now process all collected files in parallel using rayon
280+
// This is done after file discovery to avoid blocking WalkParallel threads
281+
let project_files: Vec<ProjectFile> = project_files
282+
.into_par_iter()
283+
.map(|mut file| {
284+
// Build the actual project file with owner information
285+
let built_file = self.project_file_builder.build(file.path.clone());
286+
file.owner = built_file.owner;
287+
file
288+
})
289+
.collect();
290+
276291
let teams_by_name = teams
277292
.iter()
278293
.flat_map(|team| vec![(team.name.clone(), team.clone()), (team.github_team.clone(), team.clone())])

0 commit comments

Comments
 (0)