Skip to content

Commit 70c2384

Browse files
perryqhperryqh
committed
improving project build speed
1 parent c472ed8 commit 70c2384

File tree

7 files changed

+290
-211
lines changed

7 files changed

+290
-211
lines changed

‎Cargo.lock‎

Lines changed: 10 additions & 9 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Cargo.toml‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,10 +10,11 @@ debug = true
1010
clap = { version = "4.5.20", features = ["derive"] }
1111
clap_derive = "4.5.18"
1212
error-stack = "0.5.0"
13-
glob-match = "0.2.1"
13+
fast-glob = "0.4.0"
1414
ignore = "0.4.23"
1515
itertools = "0.13.0"
1616
path-clean = "1.0.1"
17+
lazy_static = "1.5.0"
1718
rayon = "1.10.0"
1819
regex = "1.11.1"
1920
serde = { version = "1.0.214", features = ["derive"] }

‎src/common_test.rs‎

Lines changed: 5 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -10,7 +10,7 @@ pub mod tests {
1010

1111
use tempfile::tempdir;
1212

13-
use crate::{ownership::Ownership, project::Project};
13+
use crate::{ownership::Ownership, project_builder::ProjectBuilder};
1414

1515
macro_rules! ownership {
1616
($($test_files:expr),+) => {{
@@ -110,13 +110,15 @@ pub mod tests {
110110
let config = serde_yaml::from_reader(config_file)?;
111111

112112
let codeowners_file_path = &test_config.temp_dir_path.join(".github/CODEOWNERS");
113-
let project = Project::build(&test_config.temp_dir_path, codeowners_file_path, &config)?;
113+
let mut builder = ProjectBuilder::new(&config, test_config.temp_dir_path.clone(), codeowners_file_path.clone());
114+
let project = builder.build()?;
114115
let ownership = Ownership::build(project);
115116
if test_config.generate_codeowners {
116117
std::fs::write(codeowners_file_path, ownership.generate_file())?;
117118
}
118119
// rebuild project to ensure new codeowners file is read
119-
let project = Project::build(&test_config.temp_dir_path, codeowners_file_path, &config)?;
120+
let mut builder = ProjectBuilder::new(&config, test_config.temp_dir_path.clone(), codeowners_file_path.clone());
121+
let project = builder.build()?;
120122
Ok(Ownership::build(project))
121123
}
122124

‎src/main.rs‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,6 @@
11
use ownership::{FileOwner, Ownership};
22

3-
use crate::project::Project;
3+
use crate::project_builder::ProjectBuilder;
44
use clap::{Parser, Subcommand};
55
use core::fmt;
66
use error_stack::{Context, Result, ResultExt};
@@ -15,6 +15,7 @@ mod common_test;
1515
mod config;
1616
mod ownership;
1717
mod project;
18+
mod project_builder;
1819

1920
#[derive(Subcommand, Debug)]
2021
enum Command {
@@ -114,7 +115,9 @@ fn cli() -> Result<(), Error> {
114115

115116
let config = serde_yaml::from_reader(config_file).change_context(Error::Io)?;
116117

117-
let ownership = Ownership::build(Project::build(&project_root, &codeowners_file_path, &config).change_context(Error::Io)?);
118+
let mut builder = ProjectBuilder::new(&config, project_root, codeowners_file_path.clone());
119+
let project = builder.build().change_context(Error::Io)?;
120+
let ownership = Ownership::build(project);
118121

119122
match args.command {
120123
Command::Validate => ownership.validate().change_context(Error::ValidationFailed)?,

‎src/ownership/mapper.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
use glob_match::glob_match;
1+
use fast_glob::glob_match;
22
use std::{
33
collections::HashMap,
44
fmt::{self, Display},

‎src/project.rs‎

Lines changed: 4 additions & 195 deletions
Original file line numberDiff line numberDiff line change
@@ -2,21 +2,11 @@ use core::fmt;
22
use std::{
33
collections::HashMap,
44
fmt::Display,
5-
fs::File,
6-
io::BufRead,
75
path::{Path, PathBuf},
86
};
97

108
use error_stack::{Context, Result, ResultExt};
119

12-
use ignore::WalkBuilder;
13-
use rayon::prelude::{IntoParallelIterator, ParallelIterator};
14-
use regex::Regex;
15-
use tracing::{info, instrument};
16-
17-
use crate::config::Config;
18-
use glob_match::glob_match;
19-
2010
pub struct Project {
2111
pub base_path: PathBuf,
2212
pub files: Vec<ProjectFile>,
@@ -33,7 +23,7 @@ pub struct VendoredGem {
3323
pub name: String,
3424
}
3525

36-
#[derive(Debug)]
26+
#[derive(Debug, Clone)]
3727
pub struct ProjectFile {
3828
pub owner: Option<String>,
3929
pub path: PathBuf,
@@ -49,6 +39,7 @@ pub struct Team {
4939
pub avoid_ownership: bool,
5040
}
5141

42+
#[derive(Clone, Debug)]
5243
pub struct Package {
5344
pub path: PathBuf,
5445
pub package_type: PackageType,
@@ -73,7 +64,7 @@ impl DirectoryCodeownersFile {
7364
}
7465
}
7566

76-
#[derive(PartialEq, Eq, Debug)]
67+
#[derive(PartialEq, Eq, Debug, Clone)]
7768
pub enum PackageType {
7869
Ruby,
7970
Javascript,
@@ -85,7 +76,7 @@ impl Display for PackageType {
8576
}
8677
}
8778

88-
mod deserializers {
79+
pub mod deserializers {
8980
use serde::Deserialize;
9081

9182
#[derive(Deserialize)]
@@ -155,113 +146,6 @@ impl fmt::Display for Error {
155146
impl Context for Error {}
156147

157148
impl Project {
158-
#[instrument(level = "debug", skip_all)]
159-
pub fn build(base_path: &Path, codeowners_file_path: &Path, config: &Config) -> Result<Self, Error> {
160-
info!(base_path = base_path.to_str(), "scanning project");
161-
162-
let mut owned_file_paths: Vec<PathBuf> = Vec::new();
163-
let mut packages: Vec<Package> = Vec::new();
164-
let mut teams: Vec<Team> = Vec::new();
165-
let mut vendored_gems: Vec<VendoredGem> = Vec::new();
166-
let mut directory_codeowner_files: Vec<DirectoryCodeownersFile> = Vec::new();
167-
168-
let mut builder = WalkBuilder::new(base_path);
169-
builder.hidden(false);
170-
let walkdir = builder.build();
171-
172-
for entry in walkdir {
173-
let entry = entry.change_context(Error::Io)?;
174-
175-
let absolute_path = entry.path();
176-
let relative_path = absolute_path.strip_prefix(base_path).change_context(Error::Io)?.to_owned();
177-
178-
if entry.file_type().unwrap().is_dir() {
179-
if relative_path.parent() == Some(Path::new(&config.vendored_gems_path)) {
180-
let file_name = relative_path.file_name().expect("expected a file_name");
181-
vendored_gems.push(VendoredGem {
182-
path: absolute_path.to_owned(),
183-
name: file_name.to_string_lossy().to_string(),
184-
})
185-
}
186-
187-
continue;
188-
}
189-
190-
let file_name = relative_path.file_name().expect("expected a file_name");
191-
192-
if file_name.eq_ignore_ascii_case("package.yml") && matches_globs(relative_path.parent().unwrap(), &config.ruby_package_paths) {
193-
if let Some(owner) = ruby_package_owner(absolute_path)? {
194-
packages.push(Package {
195-
path: relative_path.clone(),
196-
owner,
197-
package_type: PackageType::Ruby,
198-
})
199-
}
200-
}
201-
202-
if file_name.eq_ignore_ascii_case("package.json")
203-
&& matches_globs(relative_path.parent().unwrap(), &config.javascript_package_paths)
204-
{
205-
if let Some(owner) = javascript_package_owner(absolute_path)? {
206-
packages.push(Package {
207-
path: relative_path.clone(),
208-
owner,
209-
package_type: PackageType::Javascript,
210-
})
211-
}
212-
}
213-
214-
if file_name.eq_ignore_ascii_case(".codeowner") {
215-
let owner = std::fs::read_to_string(absolute_path).change_context(Error::Io)?;
216-
let owner = owner.trim().to_owned();
217-
218-
let relative_path = relative_path.to_owned();
219-
directory_codeowner_files.push(DirectoryCodeownersFile {
220-
path: relative_path,
221-
owner,
222-
})
223-
}
224-
225-
if matches_globs(&relative_path, &config.team_file_glob) {
226-
let file = File::open(absolute_path).change_context(Error::Io)?;
227-
let deserializer: deserializers::Team = serde_yaml::from_reader(file).change_context(Error::SerdeYaml)?;
228-
229-
teams.push(Team {
230-
path: absolute_path.to_owned(),
231-
name: deserializer.name,
232-
github_team: deserializer.github.team,
233-
owned_globs: deserializer.owned_globs,
234-
owned_gems: deserializer.ruby.map(|ruby| ruby.owned_gems).unwrap_or_default(),
235-
avoid_ownership: deserializer.github.do_not_add_to_codeowners_file,
236-
})
237-
}
238-
239-
if matches_globs(&relative_path, &config.owned_globs) && !matches_globs(&relative_path, &config.unowned_globs) {
240-
owned_file_paths.push(absolute_path.to_owned())
241-
}
242-
}
243-
244-
info!(
245-
owned_file_paths = owned_file_paths.len(),
246-
packages = packages.len(),
247-
teams = teams.len(),
248-
vendored_gems = vendored_gems.len(),
249-
"finished scanning project",
250-
);
251-
252-
let owned_files = owned_files(owned_file_paths);
253-
254-
Ok(Project {
255-
base_path: base_path.to_owned(),
256-
files: owned_files,
257-
vendored_gems,
258-
teams,
259-
packages,
260-
codeowners_file_path: codeowners_file_path.to_path_buf(),
261-
directory_codeowner_files,
262-
})
263-
}
264-
265149
pub fn get_codeowners_file(&self) -> Result<String, Error> {
266150
let codeowners_file: String = if self.codeowners_file_path.exists() {
267151
std::fs::read_to_string(&self.codeowners_file_path).change_context(Error::Io)?
@@ -301,78 +185,3 @@ impl Project {
301185
result
302186
}
303187
}
304-
305-
#[instrument(level = "debug", skip_all)]
306-
fn owned_files(owned_file_paths: Vec<PathBuf>) -> Vec<ProjectFile> {
307-
let regexp = Regex::new(r#"^(?:#|//) @team (.*)$"#).expect("error compiling regular expression");
308-
309-
info!("opening files to read ownership annotation");
310-
311-
owned_file_paths
312-
.into_par_iter()
313-
.map(|path| {
314-
let file = File::open(&path).unwrap_or_else(|_| panic!("Couldn't open {}", path.to_string_lossy()));
315-
let first_line = std::io::BufReader::new(file).lines().next().transpose();
316-
let first_line = first_line.expect("error reading first line");
317-
318-
if first_line.is_none() {
319-
return ProjectFile { path, owner: None };
320-
}
321-
322-
if let Some(first_line) = first_line {
323-
let capture = regexp.captures(&first_line);
324-
325-
if let Some(capture) = capture {
326-
let first_capture = capture.get(1);
327-
328-
if let Some(first_capture) = first_capture {
329-
return ProjectFile {
330-
path,
331-
owner: Some(first_capture.as_str().to_string()),
332-
};
333-
}
334-
}
335-
}
336-
337-
ProjectFile { path, owner: None }
338-
})
339-
.collect()
340-
}
341-
342-
fn ruby_package_owner(path: &Path) -> Result<Option<String>, Error> {
343-
let file = File::open(path).change_context(Error::Io)?;
344-
let deserializer: deserializers::RubyPackage = serde_yaml::from_reader(file).change_context(Error::SerdeYaml)?;
345-
346-
Ok(deserializer.owner)
347-
}
348-
349-
fn javascript_package_owner(path: &Path) -> Result<Option<String>, Error> {
350-
let file = File::open(path).change_context(Error::Io)?;
351-
let deserializer: deserializers::JavascriptPackage = serde_json::from_reader(file).change_context(Error::SerdeJson)?;
352-
353-
Ok(deserializer.metadata.and_then(|metadata| metadata.owner))
354-
}
355-
356-
fn matches_globs(path: &Path, globs: &[String]) -> bool {
357-
globs.iter().any(|glob| glob_match(glob, path.to_str().unwrap()))
358-
}
359-
360-
#[cfg(test)]
361-
mod tests {
362-
use super::*;
363-
364-
const OWNED_GLOB: &str = "{app,components,config,frontend,lib,packs,spec,danger,script}/**/*.{rb,arb,erb,rake,js,jsx,ts,tsx}";
365-
366-
#[test]
367-
fn test_matches_globs() {
368-
// should fail because hidden directories are ignored by glob patterns unless explicitly included
369-
assert!(matches_globs(Path::new("script/.eslintrc.js"), &[OWNED_GLOB.to_string()]));
370-
}
371-
372-
#[test]
373-
fn test_glob_match() {
374-
// Exposes bug in glob-match https://github.com/devongovett/glob-match/issues/9
375-
// should fail because hidden directories are ignored by glob patterns unless explicitly included
376-
assert!(glob_match(OWNED_GLOB, "script/.eslintrc.js"));
377-
}
378-
}

0 commit comments

Comments
 (0)