Update GenerateDocstring.py (#193)

* Update GenerateDocstring.py

* Update

* better ignore

---------

Co-authored-by: TIANYOU CHEN <42710806+CTY-git@users.noreply.github.com>

Author: codelion

patchwork/common/ignore.py

@@ -1,15 +1,39 @@
+import itertools
+from itertools import chain
+from pathlib import Path
+from typing import Iterable
+
+# switch to pure globs once https://github.com/python/cpython/issues/73435 is resolved
 IGNORE_DIRS = {
     ".git",
     ".idea",
     "__pycache__",
+    ".mvn",
+}
+
+IGNORE_EXTS_GLOBS = {
+    "*.pyc",
+    "*.pyo",
+    "*.pyd",
+    "*.whl",
+    "*.egg",
+    "*.egg-info",
+    "*.dist-info",
 }
 
-IGNORE_EXTS = {
-    ".pyc",
-    ".pyo",
-    ".pyd",
-    ".whl",
-    ".egg",
-    ".egg-info",
-    ".dist-info",
+IGNORE_FILES_GLOBS = {
+    "requirements.txt",
+    "requirements-dev.txt",
+    "requirements-test.txt",
+    "mvnw",
+    "mvnw.cmd",
+    "gradlew",
+    "gradlew.bat",
 }
+
+
+def is_ignored(file_path: Path, ignored_dirs: Iterable[str] | None = None, *globs: Iterable[str]) -> bool:
+    final_ignore_globs = globs if globs else itertools.chain(IGNORE_EXTS_GLOBS, IGNORE_FILES_GLOBS)
+    final_ignored_dirs = ignored_dirs if ignored_dirs else IGNORE_DIRS
+    return (any(file_path.match(ignore_glob) for ignore_glob in final_ignore_globs) or
+            any(ignore_dir in file_path.parts[:-1] for ignore_dir in final_ignored_dirs))

patchwork/patchflows/GenerateDocstring/GenerateDocstring.py

@@ -34,7 +34,8 @@ def __init__(self, inputs: dict):
         final_inputs["branch_prefix"] = f"{self.__class__.__name__.lower()}-"
         final_inputs["context_grouping"] = "FUNCTION"
         final_inputs["allow_overlap_contexts"] = False
-
+        final_inputs["force_code_contexts"] = final_inputs.get("rewrite_existing", False)
+        
         self.inputs: dict[str, Any] = final_inputs
 
     def run(self) -> dict:

patchwork/patchflows/GenerateDocstring/defaults.yml

@@ -1,5 +1,6 @@
 # GenerateDocstring Inputs
 base_path: .
+rewrite_existing: false
 
 # CallLLM Inputs
 # For OpenAI API use the following the select the model

patchwork/steps/ExtractCodeContexts/ExtractCodeContexts.py

@@ -3,7 +3,7 @@
 
 from patchwork.common.context_strategy.context_strategies import ContextStrategies
 from patchwork.common.context_strategy.position import Position
-from patchwork.common.ignore import IGNORE_DIRS, IGNORE_EXTS
+from patchwork.common.ignore import is_ignored
 from patchwork.logger import logger
 from patchwork.step import Step
 
@@ -81,15 +81,9 @@ def run(self) -> dict:
         if self.base_path.is_file():
             files_to_consider.append(self.base_path)
         for root, dirs, files in os.walk(self.base_path):
-            root_path = Path(root)
-            if IGNORE_DIRS.intersection(root_path.parents):
-                continue
-
             for file in files:
-                if any(file.endswith(ext) for ext in IGNORE_EXTS):
-                    continue
-                file_path = root_path / file
-                if not file_path.is_file():
+                file_path = Path(root) / file
+                if not file_path.is_file() or is_ignored(file_path):
                     continue
                 files_to_consider.append(file_path)