NEW(cpd): @W-16866826@: Implement java side CpdWrapper (#115)

Author: stephen-carter-at-sf

package.json

@@ -26,6 +26,7 @@
     "rimraf": "*"
   },
   "jest": {
+    "testTimeout": 60000,
     "coverageThreshold":  {
       "global": {
         "branches": 80,

packages/code-analyzer-pmd-engine/package.json

@@ -58,6 +58,7 @@
       "showcoverage": "npm run showcoverage-java && npm run showcoverage-typescript"
     },
     "jest": {
+      "testTimeout": 60000,
       "preset": "ts-jest",
       "testEnvironment": "node",
       "testMatch": [

packages/code-analyzer-pmd-engine/pmd-wrapper/src/main/java/com/salesforce/sfca/cpdwrapper/CpdMatch.java

@@ -0,0 +1,23 @@
+package com.salesforce.sfca.cpdwrapper;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Java object to help us build cpd results that will be serializable to json format
+ * The data structure that we will serialize is Map<String, List<CpdMatch>> which will contain matches for each language.
+ */
+public class CpdMatch {
+    public int numTokensInBlock;
+    public int numNonemptyLinesInBlock;
+    public int numBlocks;
+    public List<BlockLocation> blockLocations = new ArrayList<>();
+
+    public static class BlockLocation {
+        public String file;
+        public int startLine;
+        public int startCol;
+        public int endLine;
+        public int endCol;
+    }
+}

packages/code-analyzer-pmd-engine/pmd-wrapper/src/main/java/com/salesforce/sfca/cpdwrapper/CpdRunInputData.java

@@ -0,0 +1,13 @@
+package com.salesforce.sfca.cpdwrapper;
+
+import java.util.List;
+import java.util.Map;
+
+/**
+ * Data structure for the CpdRunner that we can deserialize the input json file into
+ */
+class CpdRunInputData {
+    public Map<String, List<String>> filesToScanPerLanguage;
+    public int minimumTokens;
+    public boolean skipDuplicateFiles;
+}

packages/code-analyzer-pmd-engine/pmd-wrapper/src/main/java/com/salesforce/sfca/cpdwrapper/CpdRunner.java

@@ -0,0 +1,134 @@
+package com.salesforce.sfca.cpdwrapper;
+
+import net.sourceforge.pmd.cpd.CPDConfiguration;
+import net.sourceforge.pmd.cpd.CpdAnalysis;
+import net.sourceforge.pmd.cpd.Mark;
+import net.sourceforge.pmd.cpd.Match;
+import net.sourceforge.pmd.lang.Language;
+import net.sourceforge.pmd.lang.document.FileLocation;
+import net.sourceforge.pmd.reporting.Report;
+import net.sourceforge.pmd.util.log.PmdReporter;
+import org.slf4j.event.Level;
+
+import javax.annotation.Nullable;
+import java.io.IOException;
+import java.nio.file.Path;
+import java.nio.file.Paths;
+import java.text.MessageFormat;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+/**
+ * Class to help us invoke CPD - once for each language that should be processed
+ */
+class CpdRunner {
+    public Map<String, List<CpdMatch>> run(CpdRunInputData runInputData) throws IOException {
+        validateRunInputData(runInputData);
+
+        Map<String, List<CpdMatch>> results = new HashMap<>();
+
+        for (Map.Entry<String, List<String>> entry : runInputData.filesToScanPerLanguage.entrySet()) {
+            String language = entry.getKey();
+            List<String> filesToScan = entry.getValue();
+            if (filesToScan.isEmpty()) {
+                continue;
+            }
+            List<Path> pathsToScan = filesToScan.stream().map(Paths::get).collect(Collectors.toList());
+            List<CpdMatch> languageMatches = runLanguage(language, pathsToScan, runInputData.minimumTokens, runInputData.skipDuplicateFiles);
+
+            if (!languageMatches.isEmpty()) {
+                results.put(language, languageMatches);
+            }
+        }
+
+        return results;
+    }
+
+    private List<CpdMatch> runLanguage(String language, List<Path> pathsToScan, int minimumTokens, boolean skipDuplicateFiles) throws IOException {
+        // Note that the name "minimumTokens" comes from the public facing documentation and the cli but
+        // behind the scenes, it maps to MinimumTileSize. To learn more about the mappings to the config, see:
+        // https://github.com/pmd/pmd/blob/main/pmd-cli/src/main/java/net/sourceforge/pmd/cli/commands/internal/CpdCommand.java
+        CPDConfiguration config = new CPDConfiguration();
+        Language cpdLanguageId = config.getLanguageRegistry().getLanguageById(language);
+        if (cpdLanguageId == null) {
+            throw new RuntimeException("The language \"" + language + "\" is not recognized by CPD.");
+        }
+        config.setOnlyRecognizeLanguage(cpdLanguageId);
+        config.setMinimumTileSize(minimumTokens);
+        config.setInputPathList(pathsToScan);
+        config.setSkipDuplicates(skipDuplicateFiles);
+        config.setReporter(new CpdErrorListener());
+
+        List<CpdMatch> cpdMatches = new ArrayList<>();
+
+        try (CpdAnalysis cpd = CpdAnalysis.create(config)) {
+            cpd.performAnalysis(report -> {
+                for (Report.ProcessingError processingError : report.getProcessingErrors()) {
+                    // We don't expect any processing errors, but if there are any, then we can push them
+                    // to stdOut so that they ultimately get logged. But we should continue as normal here.
+                    System.out.println("Unexpected CPD processing error: " + processingError.getError().getMessage());
+                }
+                for (Match match : report.getMatches()) {
+                    CpdMatch cpdMatch = new CpdMatch();
+                    cpdMatch.numBlocks = match.getMarkCount();
+                    cpdMatch.numTokensInBlock = match.getTokenCount();
+                    cpdMatch.numNonemptyLinesInBlock = match.getLineCount();
+
+                    for (Mark mark : match.getMarkSet()) {
+                        CpdMatch.BlockLocation blockLocation = new CpdMatch.BlockLocation();
+                        FileLocation location = mark.getLocation();
+                        blockLocation.file = location.getFileId().getAbsolutePath();
+                        blockLocation.startLine = location.getStartLine();
+                        blockLocation.startCol = location.getStartColumn();
+                        blockLocation.endLine = location.getEndLine();
+                        blockLocation.endCol = location.getEndColumn();
+
+                        cpdMatch.blockLocations.add(blockLocation);
+                    }
+
+                    cpdMatches.add(cpdMatch);
+                }
+            });
+        }
+
+        return cpdMatches;
+    }
+
+    private void validateRunInputData(CpdRunInputData runInputData) {
+        if (runInputData.filesToScanPerLanguage == null) {
+            throw new RuntimeException("The \"filesToScanPerLanguage\" field was not set.");
+        } else if (runInputData.filesToScanPerLanguage.isEmpty()) {
+            throw new RuntimeException(("The \"filesToScanPerLanguage\" field was found to be empty."));
+        } else if (runInputData.minimumTokens <= 0) {
+            throw new RuntimeException("The \"minimumTokens\" field was not set to a positive number.");
+        }
+    }
+}
+
+// This class simply helps us process any errors that may be thrown by CPD. By default, CPD suppresses errors so that
+// they are not thrown. So here, we look out for the errors that we care about and process it to throw a better
+// error messages. We override the logEx method in particular because all other error methods call through to logEx.
+class CpdErrorListener implements PmdReporter {
+    @Override
+    public void logEx(Level level, @javax.annotation.Nullable String s, Object[] objects, @Nullable Throwable throwable) {
+        if (throwable != null) {
+            throw new RuntimeException("CPD threw an unexpected exception:\n" + throwable.getMessage(), throwable);
+        } else if (s != null) {
+            String message = MessageFormat.format(s, objects);
+            throw new RuntimeException("CPD threw an unexpected exception:\n" + message);
+        }
+    }
+
+    // These methods aren't needed or used, but they are required to be implemented (since the interface does not give them default implementations)
+    @Override
+    public boolean isLoggable(Level level) {
+        return false;
+    }
+    @Override
+    public int numErrors() {
+        return 0;
+    }
+}

packages/code-analyzer-pmd-engine/pmd-wrapper/src/main/java/com/salesforce/sfca/cpdwrapper/CpdWrapper.java

@@ -0,0 +1,100 @@
+package com.salesforce.sfca.cpdwrapper;
+
+import java.io.FileReader;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import com.google.gson.Gson;
+
+/**
+ * Provides following commands:
+ *   RUN:
+ *     - Runs the rules provided by the input ruleset file on a set of files and writes results to a JSON file
+ *     - Invocation: java -cp {classPath} com.salesforce.sfca.cpdwrapper.CpdWrapper run {argsInputFile} {resultsOutputFile}
+ *         - {classPath} is the list of entries to add to the class path
+ *         - {argsInputFile} is a JSON file containing the input arguments for the run command.
+ *              Example:
+ *                  {
+ *                      "filesToScanPerLanguage": {
+ *                          "apex": ["/full/path/to/apex_file1.cls", "/full/path/to/apex_file2.trigger", ...],
+ *                          ...,
+ *                          "xml": ["full/path/to/xml_file1.xml", "/full/path/to/xml_file2.xml", ...]
+ *                      },
+ *                      "minimumTokens": 100,
+ *                      "skipDuplicateFiles": false
+ *                  }
+ *         - {resultsOutputFile} is a JSON file to write CPD results to.
+ *             Example:
+ *                 {
+ *                     "apex": [
+ *                         {
+ *                             "numTokensInBlock": 18,
+ *                             "numNonemptyLinesInBlock": 5,
+ *                             "numBlocks": 2,
+ *                             "blockLocations": [
+ *                                 {
+ *                                     "file": "/full/path/to/file1.cls",
+ *                                     "startLine": 1, "startCol": 1, "endLine": 5, "endCol": 2
+ *                                 },
+ *                                 {
+ *                                     "file": "/full/path/to/file2.cls",
+ *                                     "startLine": 18, "startCol": 6, "endLine": 22, "endCol": 8
+ *                                 }
+ *                             ]
+ *                         },
+ *                         ...
+ *                     ],
+ *                     "xml": ...
+ *                 }
+ */
+public class CpdWrapper {
+    public static void main(String[] args) {
+        long startTime = System.currentTimeMillis();
+        System.out.println("START OF CALL TO \"CpdWrapper\" WITH ARGUMENTS: " + String.join(" ", args));
+
+        if (args.length == 0) {
+            throw new RuntimeException("Missing arguments to CpdWrapper.");
+        } else if(args[0].equalsIgnoreCase("run")) {
+            invokeRunCommand(Arrays.copyOfRange(args, 1, args.length));
+        } else {
+            throw new RuntimeException("Bad first argument to CpdWrapper. Expected \"run\". Received: \"" + args[0] + "\"");
+        }
+
+        long endTime = System.currentTimeMillis();
+        System.out.println("END OF CALL TO \"CpdWrapper\": " + (endTime - startTime) + " milliseconds");
+    }
+
+    private static void invokeRunCommand(String[] args) {
+        if (args.length != 2) {
+            throw new RuntimeException("Invalid number of arguments following the \"run\" command. Expected 2 but received: " + args.length);
+        }
+        String argsInputFile = args[0];
+        String resultsOutputFile = args[1];
+
+        Gson gson = new Gson();
+
+        CpdRunInputData inputData;
+        try (FileReader reader = new FileReader(argsInputFile)) {
+            inputData = gson.fromJson(reader, CpdRunInputData.class);
+        } catch (Exception e) {
+            throw new RuntimeException("Could not read contents from \"" + argsInputFile + "\"", e);
+        }
+
+        CpdRunner cpdRunner = new CpdRunner();
+        Map<String, List<CpdMatch>> results;
+        try {
+            results = cpdRunner.run(inputData);
+        } catch (Exception e) {
+            throw new RuntimeException("Error while attempting to invoke CpdRunner.run: " + e.getMessage(), e);
+        }
+
+        try (FileWriter fileWriter = new FileWriter(resultsOutputFile)) {
+            gson.toJson(results, fileWriter);
+        } catch (IOException e) {
+            throw new RuntimeException(e);
+        }
+    }
+}

packages/code-analyzer-pmd-engine/pmd-wrapper/src/main/java/com/salesforce/sfca/pmdwrapper/PmdRuleDescriber.java

@@ -9,6 +9,7 @@
 import org.slf4j.event.Level;
 
 import javax.annotation.Nullable;
+import java.text.MessageFormat;
 import java.util.*;
 import java.util.regex.Pattern;
 import java.util.regex.Matcher;
@@ -152,9 +153,9 @@ private static String getLimitedDescription(Rule rule) {
     }
 }
 
-// This class simply helps us process any errors that may be thrown by PMD. By default PMD suppresses errors so that
+// This class simply helps us process any errors that may be thrown by PMD. By default, PMD suppresses errors so that
 // they are not thrown. So here, we look out for the errors that we care about and process it to throw a better
-// error messages.
+// error messages. We override the logEx method in particular because all other error methods call through to logEx.
 class PmdErrorListener implements PmdReporter {
     @Override
     public void logEx(Level level, @Nullable String s, Object[] objects, @Nullable Throwable throwable) {
@@ -174,6 +175,9 @@ public void logEx(Level level, @Nullable String s, Object[] objects, @Nullable T
                 }
             }
             throw new RuntimeException("PMD threw an unexpected exception:\n" + message, throwable);
+        } else if (s != null) {
+            String message = MessageFormat.format(s, objects);
+            throw new RuntimeException("PMD threw an unexpected exception:\n" + message);
         }
     }
 

packages/code-analyzer-pmd-engine/pmd-wrapper/src/main/java/com/salesforce/sfca/pmdwrapper/PmdWrapper.java

@@ -16,13 +16,13 @@
  * Provides following commands:
  *   DESCRIBE:
  *     - Describes the available PMD rules by writing a list of PmdRuleInfo objects to a JSON file
- *     - Invocation: java -cp {classPath} describe {outputFile} {languages}
+ *     - Invocation: java -cp {classPath} com.salesforce.sfca.pmdwrapper.PmdWrapper describe {outputFile} {languages}
  *         - {classPath} is the list of entries to add to the class path
  *         - {outputFile} is a file to write the array of PmdRuleInfo objects to in JSON format
  *         - {languages} is a comma separated list of languages associated with the rules to describe
  *   RUN:
  *     - Runs the rules provided by the input ruleset file on a set of files and writes results to a JSON file
- *     - Invocation: java -cp {classPath} run {ruleSetInputFile} {filesToScanInputFile} {resultsOutputFile}
+ *     - Invocation: java -cp {classPath} com.salesforce.sfca.pmdwrapper.PmdWrapper run {ruleSetInputFile} {filesToScanInputFile} {resultsOutputFile}
  *         - {classPath} is the list of entries to add to the class path
  *         - {ruleSetInputFile} is a PMD ruleset file that contains the rules to run
  *         - {filesToScanInputFile} is a file containing a newline separated list of files to scan