Added list of igore paths and extensions

Author: rimutaka

stackmuncher/src/cmd_munch.rs

@@ -31,7 +31,7 @@ pub(crate) async fn run(config: AppConfig) -> Result<(), ()> {
     let cached_project_report = Report::from_disk(&project_report_filename);
 
     // get and retain a copy of the full git lot to re-use in multiple places
-    let git_log = git::get_log(&config.lib_config.project_dir, None).await?;
+    let git_log = git::get_log(&config.lib_config.project_dir, None, &code_rules.ignore_paths).await?;
 
     let project_report = match Report::process_project(
         &mut code_rules,

stackmuncher_lib/src/code_rules.rs

@@ -30,6 +30,9 @@ pub struct CodeRules {
     /// Contains names of newly added munchers to assist merging multiple instances
     /// of CodeRules for parallel processing.
     pub new_munchers: Option<HashSet<String>>,
+
+    /// Compiled regex for file names and paths that should be ignored regardless of any other rules
+    pub ignore_paths: Vec<Regex>,
 }
 
 impl CodeRules {
@@ -58,6 +61,7 @@ impl CodeRules {
             // dir\foo -> foo
             file_ext_regex: Regex::new(r#"[\.\\/][a-zA-Z0-1_]+$|^[a-zA-Z0-1_]+$"#).unwrap(),
             new_munchers: None,
+            ignore_paths: crate::ignore_paths::compile_ignore_paths(),
         };
 
         // load the contents of file_type definitions one by one

stackmuncher_lib/src/git.rs

@@ -1,3 +1,4 @@
+use regex::Regex;
 use std::collections::{HashMap, HashSet};
 use std::path::Path;
 use tokio::process::Command;
@@ -209,7 +210,11 @@ pub(crate) async fn populate_blob_sha1(
 /// 100644 blob f288702d2fa16d3cdf0035b15a9fcbc552cd88e7    LICENSE
 /// 100644 blob 9da69050aa4d1f6488a258a221217a4dd9e73b71    assets/file-types/cs.json
 /// ```
-pub(crate) async fn get_all_tree_files(dir: &Path, commit_sha1: Option<String>) -> Result<HashSet<String>, ()> {
+pub(crate) async fn get_all_tree_files(
+    dir: &Path,
+    commit_sha1: Option<String>,
+    ignore_paths: &Vec<Regex>,
+) -> Result<HashSet<String>, ()> {
     // use HEAD by default
     let commit_sha1 = commit_sha1.unwrap_or("HEAD".to_owned());
 
@@ -228,21 +233,58 @@ pub(crate) async fn get_all_tree_files(dir: &Path, commit_sha1: Option<String>)
             }
         })
         .collect::<HashSet<String>>();
-    info!("Objects in the GIT tree: {}", files.len());
+    let tree_all = files.len();
+
+    // remove ignored files
+    let files = files
+        .into_iter()
+        .filter_map(|file_path| {
+            if is_in_ignore_list(ignore_paths, &file_path) {
+                None
+            } else {
+                Some(file_path)
+            }
+        })
+        .collect::<HashSet<String>>();
+
+    info!(
+        "Objects in the GIT tree: {}, ignored: {}, processing: {}",
+        tree_all,
+        tree_all - files.len(),
+        files.len(),
+    );
 
     Ok(files)
 }
 
+/// Returns TRUE if the file matches any of the ignore regex rules from `ignore_paths` module.
+#[inline]
+fn is_in_ignore_list(ignore_paths: &Vec<Regex>, file_path: &str) -> bool {
+    // check if the path is in the ignore list
+    for ignore_regex in ignore_paths {
+        if ignore_regex.is_match(file_path) {
+            debug!("Path ignored: {}", file_path);
+            return true;
+        }
+    }
+
+    false
+}
+
 /// Get the contents of the Git blob as text.
 pub(crate) async fn get_blob_contents(dir: &Path, blob_sha1: &String) -> Result<Vec<u8>, ()> {
     let blob_contents = execute_git_command(vec!["cat-file".into(), "-p".into(), blob_sha1.into()], dir, false).await?;
 
     Ok(blob_contents)
 }
 
-/// Extracts and parses GIT log into who, what, when. No de-duping or optimisation is done. All log data is copied into the structs as-is.
+/// Extracts and parses GIT log into who, what, when. Removes ignored files. No de-duping or optimisation is done. All log data is copied into the structs as-is.
 /// Merge commits are excluded.
-pub async fn get_log(repo_dir: &Path, contributor_git_identity: Option<&String>) -> Result<Vec<GitLogEntry>, ()> {
+pub async fn get_log(
+    repo_dir: &Path,
+    contributor_git_identity: Option<&String>,
+    ignore_paths: &Vec<Regex>,
+) -> Result<Vec<GitLogEntry>, ()> {
     debug!("Extracting git log");
 
     // prepare the command that may optionally include the author name to limit commits just to that contributor
@@ -285,7 +327,10 @@ pub async fn get_log(repo_dir: &Path, contributor_git_identity: Option<&String>)
             // commit d5e742de653954bfae88f0e5f6c8f0a7a5f6c437
             // save the previous commit details and start a new one
             // the very first entry will be always blank, it is remove outside the loop
-            log_entries.push(current_log_entry);
+            if current_log_entry.files.len() > 0 {
+                // do not add a commit if a commit consists entirely of ignored files or has no files for another reason
+                log_entries.push(current_log_entry);
+            }
             current_log_entry = GitLogEntry::new();
             if line.len() > 8 {
                 current_log_entry.sha1 = line[7..].to_owned();
@@ -315,7 +360,7 @@ pub async fn get_log(repo_dir: &Path, contributor_git_identity: Option<&String>)
             }
             // name/email split failed - add the entire line
             current_log_entry.author_name_email = (author.to_owned(), String::new());
-            error!("Split failed on {}", line);
+            warn!("Split failed on {}", line);
         } else if line.starts_with("Date: ") {
             // Date:   Tue Dec 22 17:43:07 2020 +0000
             if line.len() < 9 {
@@ -326,7 +371,7 @@ pub async fn get_log(repo_dir: &Path, contributor_git_identity: Option<&String>)
             trace!("Date: {}", date);
             // go to the next line if there is no date (impossible?)
             if date.is_empty() {
-                error!("Encountered a commit with no date: {}", line);
+                warn!("Encountered a commit with no date: {}", line);
                 continue;
             }
 
@@ -351,17 +396,19 @@ pub async fn get_log(repo_dir: &Path, contributor_git_identity: Option<&String>)
             // the only remaining type of data should be the list of files
             // they are not tagged or indented - the entire line is the file name with the relative path
             // file names are displayed only with --name-only option
-            trace!("Added as a file");
-            current_log_entry.files.insert(line.into());
+            if !is_in_ignore_list(ignore_paths, line) {
+                trace!("Added as a file");
+                current_log_entry.files.insert(line.into());
+            } else {
+                trace!("Ignored");
+            }
         }
     }
 
     // the very last commit has to be pushed outside the loop
     log_entries.push(current_log_entry);
-    // the very first commit is always a blank record
-    log_entries.remove(0);
 
-    debug!("Found {} commits", log_entries.len());
+    debug!("Found {} commits of interest", log_entries.len());
     Ok(log_entries)
 }
 

stackmuncher_lib/src/ignore_paths.rs

@@ -0,0 +1,52 @@
+use regex::Regex;
+
+/// Returns a list of compiled regex with the list of paths that should be ignored.
+/// Panics if any of the regex statements is incorrect.
+pub(crate) fn compile_ignore_paths() -> Vec<Regex> {
+    IGNORE_PATHS
+        .iter()
+        .map(|ignore_path| Regex::new(ignore_path).expect(&format!("Invalid IGNORE_TYPES regex: {}", ignore_path)))
+        .collect::<Vec<Regex>>()
+}
+
+#[test]
+fn test_compile_ignore_paths() {
+    assert!(compile_ignore_paths().len() > 0);
+}
+
+/// A list of path fragments, file names, file extensions as Regex.
+/// Files with the path matching any of regex from this list are ignored.
+const IGNORE_PATHS: [&str; 25] = [
+    // known framework paths
+    r#"node_modules[/\\]"#,
+    r#"angular[/\\]README\.md"#,
+    r#"package-lock\.json"#,
+    // images
+    r#"\.ico$"#,
+    r#"\.png$"#,
+    r#"\.jpg$"#,
+    r#"\.jpeg$"#,
+    r#"\.gif$"#,
+    // documents
+    r#"\.pdf$"#,
+    r#"\.doc$"#,
+    r#"\.docx$"#,
+    r#"\.txt$"#,
+    // git files
+    r#"\.gitignore$"#,
+    r#"\.gitattributes$"#,
+    r#"\.gitkeep$"#,
+    // binaries
+    r#"\.exe$"#,
+    r#"\.dll$"#,
+    r#"\.so$"#,
+    r#"\.jar$"#,
+    // archives
+    r#"\.zip$"#,
+    r#"\.rar$"#,
+    // data files
+    r#"\.csv$"#,
+    r#"\.tsv$"#,
+    r#"\.xls$"#,
+    r#"\.xlsx$"#,
+];

stackmuncher_lib/src/lib.rs

@@ -10,6 +10,7 @@ pub mod config;
 pub mod contributor;
 pub mod file_type;
 pub mod git;
+mod ignore_paths;
 pub mod kwc;
 pub mod muncher;
 pub mod processors;
@@ -36,10 +37,14 @@ impl Report {
     ) -> Result<Option<report::Report>, ()> {
         let report = report::Report::new();
 
-        let git_log = git_log.unwrap_or(git::get_log(project_dir, None).await?);
+        // get the full git log if none was supplied
+        let git_log = match git_log {
+            Some(v) => v,
+            None => git::get_log(project_dir, None, &code_rules.ignore_paths).await?,
+        };
 
         // get the list of files in the tree at HEAD
-        let all_head_files = git::get_all_tree_files(project_dir, None).await?;
+        let all_head_files = git::get_all_tree_files(project_dir, None, &code_rules.ignore_paths).await?;
         if all_head_files.len() > Report::MAX_FILES_PER_REPO {
             warn!("Repo ignored. Too many files: {}", all_head_files.len());
             return Err(());
@@ -90,12 +95,12 @@ impl Report {
             .collect::<ListOfBlobs>();
         debug!("Blobs that could not be copied from cache: {}", blobs_to_munch.len());
 
-        // remove blobs that have no munchers - there is no point even retrieving the contents
+        // remove blobs that have no munchers or should be ignored - there is no point even retrieving the contents
         let blobs_to_munch = blobs_to_munch
             .into_iter()
-            .filter_map(|(file_name, blob)| {
-                if code_rules.get_muncher(&file_name).is_some() {
-                    Some((file_name, blob))
+            .filter_map(|(file_path, blob)| {
+                if code_rules.get_muncher(&file_path).is_some() {
+                    Some((file_path, blob))
                 } else {
                     None
                 }