add pathspec library and update crawl_local_files to ignore contents of .gitignore file

Author: robertguss

requirements.txt

@@ -5,3 +5,4 @@ gitpython>=3.1.0
 google-cloud-aiplatform>=1.25.0
 google-genai>=1.9.0
 python-dotenv>=1.0.0
+pathspec>=0.11.0

utils/crawl_local_files.py

@@ -1,67 +1,115 @@
 import os
 import fnmatch
+import pathspec
 
 def crawl_local_files(directory, include_patterns=None, exclude_patterns=None, max_file_size=None, use_relative_paths=True):
     """
     Crawl files in a local directory with similar interface as crawl_github_files.
-    
+
     Args:
         directory (str): Path to local directory
         include_patterns (set): File patterns to include (e.g. {"*.py", "*.js"})
         exclude_patterns (set): File patterns to exclude (e.g. {"tests/*"})
         max_file_size (int): Maximum file size in bytes
         use_relative_paths (bool): Whether to use paths relative to directory
-        
+
     Returns:
         dict: {"files": {filepath: content}}
     """
     if not os.path.isdir(directory):
         raise ValueError(f"Directory does not exist: {directory}")
-        
+
     files_dict = {}
-    
-    for root, _, files in os.walk(directory):
+
+    # --- Load .gitignore ---
+    gitignore_path = os.path.join(directory, '.gitignore')
+    gitignore_spec = None
+    if os.path.exists(gitignore_path):
+        try:
+            with open(gitignore_path, 'r', encoding='utf-8') as f:
+                gitignore_patterns = f.readlines()
+            gitignore_spec = pathspec.PathSpec.from_lines('gitwildmatch', gitignore_patterns)
+            print(f"Loaded .gitignore patterns from {gitignore_path}")
+        except Exception as e:
+            print(f"Warning: Could not read or parse .gitignore file {gitignore_path}: {e}")
+    # --- End Load .gitignore ---
+
+    for root, dirs, files in os.walk(directory):
+        # Filter directories using .gitignore and exclude_patterns early to avoid descending
+        # Need to process dirs list *in place* for os.walk to respect it
+        excluded_dirs = set()
+        for d in dirs:
+            dirpath_rel = os.path.relpath(os.path.join(root, d), directory)
+
+            # Check against .gitignore (important for directories)
+            if gitignore_spec and gitignore_spec.match_file(dirpath_rel):
+                excluded_dirs.add(d)
+                continue # Skip further checks if gitignored
+
+            # Check against standard exclude_patterns
+            if exclude_patterns:
+                for pattern in exclude_patterns:
+                    # Match pattern against full relative path or directory name itself
+                    if fnmatch.fnmatch(dirpath_rel, pattern) or fnmatch.fnmatch(d, pattern):
+                         excluded_dirs.add(d)
+                         break
+
+        # Modify dirs in-place: remove excluded ones
+        # Iterate over a copy (.copy()) because we are modifying the list during iteration
+        for d in dirs.copy():
+            if d in excluded_dirs:
+                dirs.remove(d)
+
+        # Now process files in the non-excluded directories
         for filename in files:
             filepath = os.path.join(root, filename)
-            
+
             # Get path relative to directory if requested
             if use_relative_paths:
                 relpath = os.path.relpath(filepath, directory)
             else:
                 relpath = filepath
-                
-            # Check if file matches any include pattern
+
+            # --- Exclusion check ---
+            excluded = False
+            # 1. Check .gitignore first
+            if gitignore_spec and gitignore_spec.match_file(relpath):
+                excluded = True
+
+            # 2. Check standard exclude_patterns if not already excluded by .gitignore
+            if not excluded and exclude_patterns:
+                for pattern in exclude_patterns:
+                    if fnmatch.fnmatch(relpath, pattern):
+                        excluded = True
+                        break
+
+            # --- Inclusion Check (remains the same) ---
             included = False
             if include_patterns:
                 for pattern in include_patterns:
                     if fnmatch.fnmatch(relpath, pattern):
                         included = True
                         break
             else:
+                # If no include patterns, include everything *not excluded*
                 included = True
-                
-            # Check if file matches any exclude pattern
-            excluded = False
-            if exclude_patterns:
-                for pattern in exclude_patterns:
-                    if fnmatch.fnmatch(relpath, pattern):
-                        excluded = True
-                        break
-                        
+
+            # Skip if not included or if excluded (by either method)
             if not included or excluded:
                 continue
-                
+            # --- End Exclusion/Inclusion Logic ---
+
             # Check file size
             if max_file_size and os.path.getsize(filepath) > max_file_size:
                 continue
-                
+
             try:
                 with open(filepath, 'r', encoding='utf-8') as f:
                     content = f.read()
                 files_dict[relpath] = content
             except Exception as e:
                 print(f"Warning: Could not read file {filepath}: {e}")
-                
+
     return {"files": files_dict}
 
 if __name__ == "__main__":