perf: parallelize Remapping::get_many #314
There are no files selected for viewing
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -1,8 +1,11 @@ | ||
| use super::Remapping; | ||
| use foundry_compilers_core::utils; | ||
| use rayon::prelude::*; | ||
| use std::{ | ||
| collections::{btree_map::Entry, BTreeMap, HashSet}, | ||
| fs::FileType, | ||
| path::{Path, PathBuf}, | ||
| sync::Mutex, | ||
| }; | ||
|
|
||
| const DAPPTOOLS_CONTRACTS_DIR: &str = "src"; | ||
|
|
@@ -52,6 +55,7 @@ impl Remapping { | |
| /// which would be multiple rededications according to our rules ("governance", "protocol-v2"), | ||
| /// are unified into `@aave` by looking at their common ancestor, the root of this subdirectory | ||
| /// (`@aave`) | ||
| #[instrument(level = "trace", name = "Remapping::find_many")] | ||
| pub fn find_many(dir: &Path) -> Vec<Self> { | ||
| /// prioritize | ||
| /// - ("a", "1/2") over ("a", "1/2/3") | ||
|
|
@@ -76,41 +80,30 @@ impl Remapping { | |
| } | ||
| } | ||
|
|
||
| let is_inside_node_modules = dir.ends_with("node_modules"); | ||
| let visited_symlink_dirs = Mutex::new(HashSet::new()); | ||
|
|
||
| // iterate over all dirs that are children of the root | ||
|
Member
There was a problem hiding this comment. we dont need to follow symlinks explicitly here, right?
Member
Author
There was a problem hiding this comment. why not, lib/x can be a symlink, it should be the same logic as the inner fn |
||
| let candidates = read_dir(dir) | ||
| .filter(|(_, file_type)| file_type.is_dir()) | ||
| .collect::<Vec<_>>() | ||
| .par_iter() | ||
| .flat_map_iter(|(dir, _)| { | ||
| find_remapping_candidates( | ||
| dir, | ||
| dir, | ||
| 0, | ||
| is_inside_node_modules, | ||
| &visited_symlink_dirs, | ||
| ) | ||
| }) | ||
| .collect::<Vec<_>>(); | ||
|
|
||
| // all combined remappings from all subdirs | ||
| let mut all_remappings = BTreeMap::new(); | ||
| for candidate in candidates { | ||
| if let Some(name) = candidate.window_start.file_name().and_then(|s| s.to_str()) { | ||
| insert_prioritized(&mut all_remappings, format!("{name}/"), candidate.source_dir); | ||
| } | ||
| } | ||
|
|
||
|
|
@@ -292,45 +285,33 @@ fn is_lib_dir(dir: &Path) -> bool { | |
| } | ||
|
|
||
| /// Returns true if the file is _hidden_ | ||
| fn is_hidden(path: &Path) -> bool { | ||
| path.file_name().and_then(|p| p.to_str()).map(|s| s.starts_with('.')).unwrap_or(false) | ||
| } | ||
|
|
||
| /// Finds all remappings in the directory recursively | ||
| /// | ||
| /// Note: this supports symlinks and will short-circuit if a symlink dir has already been visited, | ||
| /// this can occur in pnpm setups: <https://github.com/foundry-rs/foundry/issues/7820> | ||
| fn find_remapping_candidates( | ||
| current_dir: &Path, | ||
| open: &Path, | ||
| current_level: usize, | ||
| is_inside_node_modules: bool, | ||
| visited_symlink_dirs: &Mutex<HashSet<PathBuf>>, | ||
| ) -> Vec<Candidate> { | ||
| trace!("find_remapping_candidates({})", current_dir.display()); | ||
|
|
||
| // this is a marker if the current root is a candidate for a remapping | ||
| let mut is_candidate = false; | ||
|
|
||
| // scan all entries in the current dir | ||
| let mut search = Vec::new(); | ||
| for (subdir, file_type) in read_dir(current_dir) { | ||
| // found a solidity file directly the current dir | ||
| if !is_candidate && file_type.is_file() && subdir.extension() == Some("sol".as_ref()) { | ||
| is_candidate = true; | ||
| } else if file_type.is_dir() { | ||
| // if the dir is a symlink to a parent dir we short circuit here | ||
| // `walkdir` will catch symlink loops, but this check prevents that we end up scanning a | ||
| // workspace like | ||
|
|
@@ -339,9 +320,9 @@ fn find_remapping_candidates( | |
| // ├── dep/node_modules | ||
| // ├── symlink to `my-package` | ||
| // ``` | ||
| if file_type.is_symlink() { | ||
|
||
| if let Ok(target) = utils::canonicalize(&subdir) { | ||
| if !visited_symlink_dirs.lock().unwrap().insert(target.clone()) { | ||
| // short-circuiting if we've already visited the symlink | ||
| return Vec::new(); | ||
| } | ||
|
|
@@ -355,40 +336,46 @@ fn find_remapping_candidates( | |
| } | ||
| } | ||
|
|
||
| // we skip commonly used subdirs that should not be searched for recursively | ||
| if !no_recurse(&subdir) { | ||
| search.push(subdir); | ||
| } | ||
| } | ||
| } | ||
|
|
||
| // all found candidates | ||
| let mut candidates = search | ||
| .par_iter() | ||
| .flat_map_iter(|subdir| { | ||
| // scan the subdirectory for remappings, but we need a way to identify nested | ||
| // dependencies like `ds-token/lib/ds-stop/lib/ds-note/src/contract.sol`, or | ||
| // `oz/{tokens,auth}/{contracts, interfaces}/contract.sol` to assign | ||
| // the remappings to their root, we use a window that lies between two barriers. If | ||
| // we find a solidity file within a window, it belongs to the dir that opened the | ||
| // window. | ||
|
|
||
| // check if the subdir is a lib barrier, in which case we open a new window | ||
| if is_lib_dir(subdir) { | ||
| find_remapping_candidates( | ||
| subdir, | ||
| subdir, | ||
| current_level + 1, | ||
| is_inside_node_modules, | ||
| visited_symlink_dirs, | ||
| ) | ||
| } else { | ||
| // continue scanning with the current window | ||
| find_remapping_candidates( | ||
| subdir, | ||
| open, | ||
| current_level, | ||
| is_inside_node_modules, | ||
| visited_symlink_dirs, | ||
| ) | ||
| } | ||
| }) | ||
| .collect::<Vec<_>>(); | ||
|
|
||
| // need to find the actual next window in the event `open` is a lib dir | ||
| let window_start = next_nested_window(open, current_dir); | ||
| // finally, we need to merge, adjust candidates from the same level and open window | ||
|
|
@@ -425,6 +412,19 @@ fn find_remapping_candidates( | |
| candidates | ||
| } | ||
|
|
||
| fn read_dir(dir: &Path) -> impl Iterator<Item = (PathBuf, FileType)> { | ||
| std::fs::read_dir(dir) | ||
| .into_iter() | ||
| .flatten() | ||
| .filter_map(Result::ok) | ||
| .filter_map(|e| Some((e.path(), e.file_type().ok()?))) | ||
| .filter(|(p, _)| !is_hidden(p)) | ||
| } | ||
|
|
||
| fn no_recurse(dir: &Path) -> bool { | ||
| dir.ends_with("tests") || dir.ends_with("test") || dir.ends_with("demo") | ||
| } | ||
|
|
||
| /// Counts the number of components between `root` and `current` | ||
| /// `dir_distance("root/a", "root/a/b/c") == 2` | ||
| fn dir_distance(root: &Path, current: &Path) -> usize { | ||
|
|
||
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
findmodule and related functions are gated by "walkdir", don't wanna change feature gateThere was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
makes sense