Adds per codemod Includes/Excludes (#438)

Codemods now have their own includes/excludes rules and a `supports`
method.

Includes/Excludes rules are specified by `PathMatcher` patterns that are
relative to the repository root.

Before running on a given file, codemods will now perform two checks:
includes/excludes and `supports`.

Codemod specific Includes/excludes check can be overridden by global
configuration from the command line (i.e. with `--path-include` flag).

`supports` checks the bare-minimum conditions that must be satisfied
before the codemod attempts to run. For example,
javaparser codemods will check if the file is a java file and sarif
based codemods will check if there is any results for the file.

Refactored some scattered file checks in codemods into the `supports`
method. For example, the `VerbTamperingCodemod` will now check if the
file is a `web.xml` file with the `supports` method instead of the
`visit`. This has the benefit of avoiding the generation of empty codetf
reports.

Author: andrecsilva

core-codemods/src/intTest/java/io/codemodder/integration/WebGoat822Test.java

@@ -26,6 +26,29 @@ final class WebGoat822Test extends GitRepositoryTest {
     super("https://github.com/WebGoat/WebGoat", "main", "e75cfbeb110e3d3a2ca3c8fee2754992d89c419d");
   }
 
+  private static final String testPathIncludes =
+      "**.java,"
+          + "**/*.java,"
+          + "pom.xml,"
+          + "**/pom.xml,"
+          + "**.jsp,"
+          + "**/*.jsp,"
+          + "web.xml,"
+          + "**/web.xml,"
+          + ".github/workflows/*.yml,"
+          + ".github/workflows/*.yaml";
+
+  private static final String testPathExcludes =
+      "**/test/**,"
+          + "**/testFixtures/**,"
+          + "**/*Test.java,"
+          + "**/intTest/**,"
+          + "**/tests/**,"
+          + "**/target/**,"
+          + "**/build/**,"
+          + "**/.mvn/**,"
+          + ".mvn/**";
+
   @Test
   void it_injects_dependency_even_when_no_poms_included() throws Exception {
 
@@ -83,7 +106,13 @@ void it_injects_dependency_even_when_no_poms_included() throws Exception {
   void it_transforms_webgoat_normally() throws Exception {
     DefaultCodemods.main(
         new String[] {
-          "--output", outputFile.getPath(), "--verbose", "--dont-exit", repoDir.getPath()
+          "--output",
+          outputFile.getPath(),
+          "--verbose",
+          "--dont-exit",
+          repoDir.getPath(),
+          "--path-include=" + testPathIncludes,
+          "--path-exclude=" + testPathExcludes,
         });
 
     ObjectMapper objectMapper = new ObjectMapper();
@@ -121,6 +150,8 @@ void it_transforms_webgoat_with_codeql() throws Exception {
           "--sarif",
           "src/test/resources/webgoat_v8.2.2_codeql.sarif",
           "--dont-exit",
+          "--path-include=" + testPathIncludes,
+          "--path-exclude=" + testPathExcludes,
           repoDir.getPath()
         });
 

core-codemods/src/main/java/io/codemodder/codemods/AddMissingI18nCodemod.java

@@ -53,16 +53,18 @@ private List<Language> putInPreferredOrder(final List<Language> languages) {
     return orderedLanguages;
   }
 
+  @Override
+  public boolean supports(final Path file) {
+    return getPropertyFilePrefix(file.getFileName().toString()).isPresent();
+  }
+
   @Override
   public CodemodFileScanningResult visitFile(final CodemodInvocationContext context)
       throws IOException {
     Path path = context.path();
     String fileName = path.getFileName().toString();
+    // The supports check will guarantee that this won't be empty
     Optional<String> prefix = getPropertyFilePrefix(fileName);
-    if (prefix.isEmpty()) {
-      // doesn't look like a i18n properties file
-      return CodemodFileScanningResult.none();
-    }
     return doVisitFile(context, path, prefix.get());
   }
 

core-codemods/src/main/java/io/codemodder/codemods/DefaultCodemods.java

@@ -57,6 +57,7 @@ public static List<Class<? extends CodeChanger>> asList() {
         SimplifyRestControllerAnnotationsCodemod.class,
         SubstituteReplaceAllCodemod.class,
         SonarXXECodemod.class,
+        SonarSQLInjectionCodemod.class,
         SQLParameterizerCodemod.class,
         SSRFCodemod.class,
         StackTraceExposureCodemod.class,

core-codemods/src/main/java/io/codemodder/codemods/JSPScriptletXSSCodemod.java

@@ -1,7 +1,9 @@
 package io.codemodder.codemods;
 
 import io.codemodder.*;
+import java.nio.file.Path;
 import java.util.List;
+import java.util.Set;
 import java.util.regex.Pattern;
 
 /**
@@ -19,12 +21,12 @@
     reviewGuidance = ReviewGuidance.MERGE_WITHOUT_REVIEW)
 public final class JSPScriptletXSSCodemod extends RegexFileChanger {
 
+  private IncludesExcludesPattern includesExcludesPattern;
+
   public JSPScriptletXSSCodemod() {
-    super(
-        path -> path.getFileName().toString().toLowerCase().endsWith(".jsp"),
-        scriptlet,
-        true,
-        List.of(DependencyGAV.OWASP_XSS_JAVA_ENCODER));
+    super(scriptlet, true, List.of(DependencyGAV.OWASP_XSS_JAVA_ENCODER));
+    this.includesExcludesPattern =
+        new IncludesExcludesPattern.Default(Set.of("**.[jJ][sS][pP]"), Set.of());
   }
 
   @Override
@@ -34,8 +36,18 @@ public String getReplacementFor(final String matchingSnippet) {
     return "<%=org.owasp.encoder.Encode.forHtml(" + codeWithinScriptlet + ")%>";
   }
 
+  @Override
+  public boolean supports(final Path file) {
+    return file.getFileName().toString().toLowerCase().endsWith(".jsp");
+  }
+
   private static final Pattern scriptlet =
       Pattern.compile(
           "<%(\\s*)=(\\s*)request(\\s*).(\\s*)get((Header|Parameter)(\\s*)\\((\\s*)\".*\"(\\s*)\\)|QueryString\\((\\s*)\\))(\\s*)%>",
           Pattern.MULTILINE);
+
+  @Override
+  public IncludesExcludesPattern getIncludesExcludesPattern() {
+    return includesExcludesPattern;
+  }
 }

core-codemods/src/main/java/io/codemodder/codemods/MavenSecureURLCodemod.java

@@ -126,7 +126,7 @@ private CodemodFileScanningResult processXml(final Path file, final List<Result>
   }
 
   /*
-   * Change contents of the {@code url} tag if it it uses an insecure protocol.
+   * Change contents of the {@code url} tag if it uses an insecure protocol.
    */
   private static void handle(
       final XMLEventReader xmlEventReader,

core-codemods/src/main/java/io/codemodder/codemods/SpringAbsoluteCookieTimeoutCodemod.java

@@ -46,13 +46,16 @@ public SpringAbsoluteCookieTimeoutCodemod(
     this.safeDuration = parseExistingValueFromLine(count, units);
   }
 
+  @Override
+  public boolean supports(final Path file) {
+    return "application.properties".equalsIgnoreCase(file.getFileName().toString());
+  }
+
   @Override
   public CodemodFileScanningResult visitFile(final CodemodInvocationContext context)
       throws IOException {
     Path path = context.path();
-    if (!"application.properties".equalsIgnoreCase(path.getFileName().toString())) {
-      return CodemodFileScanningResult.none();
-    } else if (!inExpectedDir(context.codeDirectory().asPath().relativize(path))) {
+    if (!inExpectedDir(context.codeDirectory().asPath().relativize(path))) {
       return CodemodFileScanningResult.none();
     }
 

core-codemods/src/main/java/io/codemodder/codemods/VerbTamperingCodemod.java

@@ -36,9 +36,6 @@ public VerbTamperingCodemod(final XPathStreamProcessor processor) {
   public CodemodFileScanningResult visitFile(final CodemodInvocationContext context)
       throws IOException {
     Path file = context.path();
-    if (!"web.xml".equalsIgnoreCase(file.getFileName().toString())) {
-      return CodemodFileScanningResult.none();
-    }
     try {
       List<CodemodChange> changes = processWebXml(context, file);
       return CodemodFileScanningResult.withOnlyChanges(changes);
@@ -107,4 +104,14 @@ public String getIndividualChangeDescription(final Path filePath, final CodemodC
   public List<CodeTFReference> getReferences() {
     return reporter.getReferences().stream().map(u -> new CodeTFReference(u, u)).toList();
   }
+
+  @Override
+  public IncludesExcludesPattern getIncludesExcludesPattern() {
+    return new IncludesExcludesPattern.Default(Set.of("{,**/}[wW][eE][bB].[xX][mM][lL]"), Set.of());
+  }
+
+  @Override
+  public boolean supports(final Path file) {
+    return "web.xml".equalsIgnoreCase(file.getFileName().toString());
+  }
 }

framework/codemodder-base/src/main/java/io/codemodder/CLI.java

@@ -344,16 +344,23 @@ public Integer call() throws IOException {
 
       // get path includes/excludes
       List<String> pathIncludes = this.pathIncludes;
+      List<String> pathExcludes = this.pathExcludes;
+
+      // If no path includes/excludes were specified, relegate filtering to each codemod
+      boolean useGlobalIncludesExcludes = pathIncludes != null || pathExcludes != null;
+      IncludesExcludes includesExcludes = IncludesExcludes.any();
       if (pathIncludes == null) {
-        pathIncludes = defaultPathIncludes;
+        pathIncludes = List.of("**");
       }
 
-      List<String> pathExcludes = this.pathExcludes;
       if (pathExcludes == null) {
-        pathExcludes = defaultPathExcludes;
+        pathExcludes = List.of();
+      }
+
+      if (useGlobalIncludesExcludes) {
+        includesExcludes =
+            IncludesExcludes.withSettings(projectDirectory, pathIncludes, pathExcludes);
       }
-      IncludesExcludes includesExcludes =
-          IncludesExcludes.withSettings(projectDirectory, pathIncludes, pathExcludes);
 
       log.debug("including paths: {}", pathIncludes);
       log.debug("excluding paths: {}", pathExcludes);
@@ -431,19 +438,35 @@ public Integer call() throws IOException {
       FileCache fileCache = FileCache.createDefault(maxFileCacheSize);
 
       for (CodemodIdPair codemod : codemods) {
-        CodemodExecutor codemodExecutor =
-            new DefaultCodemodExecutor(
-                projectPath,
-                includesExcludes,
-                codemod,
-                projectProviders,
-                codeTFProviders,
-                fileCache,
-                javaParserFacade,
-                encodingDetector,
-                maxFileSize,
-                maxFiles,
-                maxWorkers);
+        CodemodExecutor codemodExecutor;
+        if (useGlobalIncludesExcludes) {
+          codemodExecutor =
+              new DefaultCodemodExecutor(
+                  projectPath,
+                  includesExcludes,
+                  codemod,
+                  projectProviders,
+                  codeTFProviders,
+                  fileCache,
+                  javaParserFacade,
+                  encodingDetector,
+                  maxFileSize,
+                  maxFiles,
+                  maxWorkers);
+        } else {
+          codemodExecutor =
+              new DefaultCodemodExecutor(
+                  projectPath,
+                  codemod,
+                  projectProviders,
+                  codeTFProviders,
+                  fileCache,
+                  javaParserFacade,
+                  encodingDetector,
+                  maxFileSize,
+                  maxFiles,
+                  maxWorkers);
+        }
 
         log.info("running codemod: {}", codemod.getId());
         CodeTFResult result = codemodExecutor.execute(filePaths);

framework/codemodder-base/src/main/java/io/codemodder/CodeChanger.java

@@ -22,6 +22,21 @@ public interface CodeChanger {
   /** A description of an individual change made by this codemod. */
   String getIndividualChangeDescription(final Path filePath, final CodemodChange change);
 
+  /**
+   * A list of paths patterns requested or rejected by the codemod. Those patterns are treated as
+   * relative to the repository root. These patterns should follow the {@link
+   * java.nio.file.PathMatcher} specification. These patterns can be overridden by global patterns.
+   */
+  default IncludesExcludesPattern getIncludesExcludesPattern() {
+    return IncludesExcludesPattern.getAnyMatcher();
+  }
+
+  /**
+   * A predicate which dictates if the file should be inspected by the codemod. This cannot be
+   * overridden and should always pass before executing the codemod.
+   */
+  boolean supports(final Path file);
+
   /**
    * A lifecycle event that is called before any files are processed. This is a good place to short
    * circuit if you don't have the necessary resources (e.g., SARIF).

framework/codemodder-base/src/main/java/io/codemodder/CompositeJavaParserChanger.java

@@ -3,6 +3,7 @@
 import com.github.javaparser.ast.CompilationUnit;
 import io.codemodder.codetf.UnfixedFinding;
 import io.codemodder.javaparser.JavaParserChanger;
+import java.nio.file.Path;
 import java.util.ArrayList;
 import java.util.Arrays;
 import java.util.List;
@@ -29,6 +30,17 @@ protected CompositeJavaParserChanger(final JavaParserChanger... changers) {
     this.changers = Arrays.asList(Objects.requireNonNull(changers));
   }
 
+  @Override
+  public IncludesExcludesPattern getIncludesExcludesPattern() {
+    // The first changer will dictate which files this composition accepts
+    return changers.get(0).getIncludesExcludesPattern();
+  }
+
+  @Override
+  public boolean supports(final Path file) {
+    return changers.stream().anyMatch(c -> c.supports(file));
+  }
+
   @Override
   public CodemodFileScanningResult visit(
       final CodemodInvocationContext context, final CompilationUnit cu) {
@@ -44,9 +56,4 @@ public CodemodFileScanningResult visit(
 
     return CodemodFileScanningResult.from(changes, unfixedFindings);
   }
-
-  @Override
-  public boolean shouldRun() {
-    return changers.stream().anyMatch(CodeChanger::shouldRun);
-  }
 }