Support included/excluded patterns in LocalFile (#133)

Author: badmonster0

Cargo.toml

@@ -56,3 +56,4 @@ tree-sitter-python = "0.23.6"
 tree-sitter-javascript = "0.23.1"
 tree-sitter-typescript = "0.23.2"
 tree-sitter-md = "0.3.2"
+globset = "0.4.16"

python/cocoindex/sources.py

@@ -8,3 +8,11 @@ class LocalFile(op.SourceSpec):
 
     path: str
     binary: bool = False
+
+    # If provided, only files matching these patterns will be included.
+    # See https://docs.rs/globset/latest/globset/index.html for the syntax of the patterns.
+    included_patterns: list[str] | None = None
+
+    # If provided, files matching these patterns will be excluded.
+    # See https://docs.rs/globset/latest/globset/index.html for the syntax of the patterns.
+    excluded_patterns: list[str] | None = None

src/ops/sources/local_file.rs

@@ -1,3 +1,4 @@
+use globset::{Glob, GlobSet, GlobSetBuilder};
 use log::warn;
 use std::{path::PathBuf, sync::Arc};
 
@@ -7,29 +8,43 @@ use crate::{fields_value, ops::sdk::*};
 pub struct Spec {
     path: String,
     binary: bool,
+    included_patterns: Option<Vec<String>>,
+    excluded_patterns: Option<Vec<String>>,
 }
 
 struct Executor {
     root_path_str: String,
     root_path: PathBuf,
     binary: bool,
+    included_glob_set: Option<GlobSet>,
+    excluded_glob_set: Option<GlobSet>,
 }
 
 impl Executor {
     async fn traverse_dir(&self, dir_path: &PathBuf, result: &mut Vec<KeyValue>) -> Result<()> {
         for entry in std::fs::read_dir(dir_path)? {
             let entry = entry?;
             let path = entry.path();
-            if path.is_dir() {
-                Box::pin(self.traverse_dir(&path, result)).await?;
-            } else {
-                if let Some(file_name) = path.to_str() {
-                    result.push(KeyValue::Str(Arc::from(
-                        &file_name[self.root_path_str.len() + 1..],
-                    )));
-                } else {
-                    warn!("Skipped ill-formed file path: {}", path.display());
+            if let Some(file_name) = path.to_str() {
+                let relative_path = &file_name[self.root_path_str.len() + 1..];
+                if self
+                    .excluded_glob_set
+                    .as_ref()
+                    .map_or(false, |glob_set| glob_set.is_match(relative_path))
+                {
+                    continue;
                 }
+                if path.is_dir() {
+                    Box::pin(self.traverse_dir(&path, result)).await?;
+                } else if self
+                    .included_glob_set
+                    .as_ref()
+                    .map_or(true, |glob_set| glob_set.is_match(relative_path))
+                {
+                    result.push(KeyValue::Str(Arc::from(relative_path)));
+                }
+            } else {
+                warn!("Skipped ill-formed file path: {}", path.display());
             }
         }
         Ok(())
@@ -102,6 +117,16 @@ impl SourceFactoryBase for Factory {
             root_path_str: spec.path.clone(),
             root_path: PathBuf::from(spec.path),
             binary: spec.binary,
+            included_glob_set: spec.included_patterns.map(build_glob_set).transpose()?,
+            excluded_glob_set: spec.excluded_patterns.map(build_glob_set).transpose()?,
         }))
     }
 }
+
+fn build_glob_set(patterns: Vec<String>) -> Result<GlobSet> {
+    let mut builder = GlobSetBuilder::new();
+    for pattern in patterns {
+        builder.add(Glob::new(pattern.as_str())?);
+    }
+    Ok(builder.build()?)
+}