Enable per-codemod configuration of included files (#480)

Author: drdavella

src/codemodder/cli.py

@@ -3,7 +3,7 @@
 import sys
 
 from codemodder import __version__
-from codemodder.code_directory import DEFAULT_EXCLUDED_PATHS, DEFAULT_INCLUDED_PATHS
+from codemodder.code_directory import DEFAULT_EXCLUDED_PATHS
 from codemodder.logging import OutputFormat, logger
 from codemodder.registry import CodemodRegistry
 
@@ -149,7 +149,7 @@ def parse_args(argv, codemod_registry: CodemodRegistry):
     parser.add_argument(
         "--path-include",
         action=CsvListAction,
-        default=DEFAULT_INCLUDED_PATHS,
+        default=[],
         help="Comma-separated set of UNIX glob patterns to include",
     )
     parser.add_argument(

src/codemodder/codemodder.py

@@ -186,13 +186,17 @@ def run(original_args) -> int:
         sast_only=argv.sonar_issues_json or argv.sarif,
     )
 
+    included_paths = argv.path_include or codemod_registry.default_include_paths
+
     log_section("setup")
     log_list(logging.INFO, "running", codemods_to_run, predicate=lambda c: c.id)
-    log_list(logging.INFO, "including paths", argv.path_include)
+    log_list(logging.INFO, "including paths", included_paths)
     log_list(logging.INFO, "excluding paths", argv.path_exclude)
 
     files_to_analyze: list[Path] = match_files(
-        context.directory, argv.path_exclude, argv.path_include
+        context.directory,
+        argv.path_exclude,
+        included_paths,
     )
 
     full_names = [str(path) for path in files_to_analyze]

src/codemodder/codemods/base_codemod.py

@@ -67,18 +67,21 @@ class BaseCodemod(metaclass=ABCMeta):
     _metadata: Metadata
     detector: BaseDetector | None
     transformer: BaseTransformerPipeline
+    default_extensions: list[str] | None
 
     def __init__(
         self,
         *,
         metadata: Metadata,
         detector: BaseDetector | None = None,
         transformer: BaseTransformerPipeline,
+        default_extensions: list[str] | None = None,
     ):
         # Metadata should only be accessed via properties
         self._metadata = metadata
         self.detector = detector
         self.transformer = transformer
+        self.default_extensions = default_extensions or [".py"]
 
     @property
     @abstractmethod
@@ -154,6 +157,16 @@ def _apply(
             else None
         )
 
+        files_to_analyze = (
+            [
+                path
+                for path in files_to_analyze
+                if path.suffix in self.default_extensions
+            ]
+            if self.default_extensions
+            else files_to_analyze
+        )
+
         process_file = functools.partial(
             self._process_file, context=context, results=results, rules=rules
         )

src/codemodder/registry.py

@@ -1,8 +1,10 @@
 from __future__ import annotations
 
+import os
 import re
 from dataclasses import dataclass
 from importlib.metadata import entry_points
+from itertools import chain
 from typing import TYPE_CHECKING, Optional
 
 from codemodder.logging import logger
@@ -31,10 +33,12 @@ class CodemodCollection:
 class CodemodRegistry:
     _codemods_by_name: dict[str, BaseCodemod]
     _codemods_by_id: dict[str, BaseCodemod]
+    _default_include_paths: set[str]
 
     def __init__(self):
         self._codemods_by_name = {}
         self._codemods_by_id = {}
+        self._default_include_paths = set()
 
     @property
     def names(self):
@@ -48,11 +52,23 @@ def ids(self):
     def codemods(self):
         return list(self._codemods_by_name.values())
 
+    @property
+    def default_include_paths(self) -> list[str]:
+        return list(self._default_include_paths)
+
     def add_codemod_collection(self, collection: CodemodCollection):
         for codemod in collection.codemods:
             wrapper = codemod() if isinstance(codemod, type) else codemod
             self._codemods_by_name[wrapper.name] = wrapper
             self._codemods_by_id[wrapper.id] = wrapper
+            self._default_include_paths.update(
+                chain(
+                    *[
+                        (f"*{ext}", os.path.join("**", f"*{ext}"))
+                        for ext in wrapper.default_extensions
+                    ]
+                )
+            )
 
     def match_codemods(
         self,

src/core_codemods/api/core_codemod.py

@@ -34,9 +34,15 @@ def __init__(
         metadata: Metadata,
         detector: BaseDetector | None = None,
         transformer: BaseTransformerPipeline,
+        default_extensions: list[str] | None = None,
         requested_rules: list[str] | None = None,
     ):
-        super().__init__(metadata=metadata, detector=detector, transformer=transformer)
+        super().__init__(
+            metadata=metadata,
+            detector=detector,
+            transformer=transformer,
+            default_extensions=default_extensions,
+        )
         self.requested_rules = [self.name]
         if requested_rules:
             self.requested_rules.extend(requested_rules)

tests/codemods/test_base_codemod.py

@@ -1,8 +1,12 @@
+from pathlib import Path
+
 import libcst as cst
 import mock
+import pytest
 from libcst.codemod import CodemodContext
 
 from codemodder.codemods.api import Metadata, ReviewGuidance, SimpleCodemod
+from core_codemods.api import CoreCodemod
 
 
 class DoNothingCodemod(SimpleCodemod):
@@ -32,3 +36,21 @@ def run_and_assert(self, input_code, expected_output):
     def test_empty_results(self):
         input_code = """print('Hello World')"""
         self.run_and_assert(input_code, input_code)
+
+
+@pytest.mark.parametrize("ext,call_count", [(".py", 2), (".txt", 1), (".js", 1)])
+def test_filter_apply_by_extension(mocker, ext, call_count):
+    process_file = mocker.patch("core_codemods.api.CoreCodemod._process_file")
+
+    codemod = CoreCodemod(
+        metadata=mocker.MagicMock(),
+        transformer=mocker.MagicMock(),
+        default_extensions=[ext],
+    )
+
+    codemod.apply(
+        mocker.MagicMock(),
+        [Path("file.py"), Path("file.txt"), Path("file.js"), Path("file2.py")],
+    )
+
+    assert process_file.call_count == call_count

tests/test_registry.py

@@ -0,0 +1,20 @@
+from codemodder.registry import CodemodCollection, CodemodRegistry
+
+
+def test_default_extensions(mocker):
+    registry = CodemodRegistry()
+    assert registry.default_include_paths == []
+
+    CodemodA = mocker.MagicMock(default_extensions=[".py"])
+    CodemodB = mocker.MagicMock(default_extensions=[".py", ".txt"])
+
+    registry.add_codemod_collection(
+        CodemodCollection(origin="origin", codemods=[CodemodA, CodemodB])
+    )
+
+    assert sorted(registry.default_include_paths) == [
+        "**/*.py",
+        "**/*.txt",
+        "*.py",
+        "*.txt",
+    ]