Skip to content

CHANGE(cpd): @W-17386401@: Update cpd engine's minimum_tokens field to be an object with values per language #153

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
stephen-carter-at-sf merged 1 commit into from
Dec 9, 2024
Merged

CHANGE(cpd): @W-17386401@: Update cpd engine's minimum_tokens field to be an object with values per language #153

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 6 additions & 2 deletions ...engine/pmd-cpd-wrappers/src/main/java/com/salesforce/sfca/cpdwrapper/CpdRunInputData.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,11 @@
* Data structure for the CpdRunner that we can deserialize the input json file into
*/
class CpdRunInputData {
public Map<String, List<String>> filesToScanPerLanguage;
public int minimumTokens;
public Map<String, LanguageSpecificRunData> runDataPerLanguage;
public boolean skipDuplicateFiles;
}

class LanguageSpecificRunData {
public List<String> filesToScan;
public int minimumTokens;
}
42 changes: 24 additions & 18 deletions ...r-pmd-engine/pmd-cpd-wrappers/src/main/java/com/salesforce/sfca/cpdwrapper/CpdRunner.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,10 +18,7 @@
import java.nio.file.Path;
import java.nio.file.Paths;
import java.text.MessageFormat;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.*;
import java.util.stream.Collectors;

/**
Expand All @@ -33,18 +30,15 @@ class CpdRunner {
public Map<String, CpdLanguageRunResults> run(CpdRunInputData runInputData) throws IOException {
validateRunInputData(runInputData);

List<String> languagesToProcess = runInputData.filesToScanPerLanguage.entrySet().stream()
.filter(entry -> !entry.getValue().isEmpty()) // Keep only non-empty lists
.map(Map.Entry::getKey)
.collect(Collectors.toList());

List<String> languagesToProcess = new ArrayList<>(runInputData.runDataPerLanguage.keySet());
progressReporter.initialize(languagesToProcess);

Map<String, CpdLanguageRunResults> results = new HashMap<>();
for (String language : languagesToProcess) {
List<String> filesToScan = runInputData.filesToScanPerLanguage.get(language);
List<Path> pathsToScan = filesToScan.stream().map(Paths::get).collect(Collectors.toList());
CpdLanguageRunResults languageRunResults = runLanguage(language, pathsToScan, runInputData.minimumTokens, runInputData.skipDuplicateFiles);
LanguageSpecificRunData languageSpecificRunData = runInputData.runDataPerLanguage.get(language);
List<Path> pathsToScan = languageSpecificRunData.filesToScan.stream().map(Paths::get).collect(Collectors.toList());
CpdLanguageRunResults languageRunResults = runLanguage(
language, pathsToScan, languageSpecificRunData.minimumTokens, runInputData.skipDuplicateFiles);
if (!languageRunResults.matches.isEmpty() || !languageRunResults.processingErrors.isEmpty()) {
results.put(language, languageRunResults);
}
Expand Down Expand Up @@ -126,12 +120,24 @@ private CpdLanguageRunResults runLanguage(String language, List<Path> pathsToSca
}

private void validateRunInputData(CpdRunInputData runInputData) {
if (runInputData.filesToScanPerLanguage == null) {
throw new RuntimeException("The \"filesToScanPerLanguage\" field was not set.");
} else if (runInputData.filesToScanPerLanguage.isEmpty()) {
throw new RuntimeException(("The \"filesToScanPerLanguage\" field was found to be empty."));
} else if (runInputData.minimumTokens <= 0) {
throw new RuntimeException("The \"minimumTokens\" field was not set to a positive number.");
if (runInputData.runDataPerLanguage == null) {
throw new RuntimeException("The \"runDataPerLanguage\" field was not set.");
}

Set<Map.Entry<String, LanguageSpecificRunData>> entries = runInputData.runDataPerLanguage.entrySet();
if (entries.isEmpty()) {
throw new RuntimeException("The \"runDataPerLanguage\" field didn't have any languages listed.");
}

for (Map.Entry<String, LanguageSpecificRunData> entry: entries) {
String language = entry.getKey();
LanguageSpecificRunData languageSpecificRunData = entry.getValue();

if (languageSpecificRunData.filesToScan == null || languageSpecificRunData.filesToScan.isEmpty()) {
throw new RuntimeException(("The \"filesToScan\" field was missing or empty for language: " + language));
} else if (languageSpecificRunData.minimumTokens <= 0) {
throw new RuntimeException("The \"minimumTokens\" field was not set to a positive number for language: " + language);
}
}
}

Expand Down
13 changes: 9 additions & 4 deletions ...-pmd-engine/pmd-cpd-wrappers/src/main/java/com/salesforce/sfca/cpdwrapper/CpdWrapper.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -17,12 +17,17 @@
* - {argsInputFile} is a JSON file containing the input arguments for the run command.
* Example:
* {
* "filesToScanPerLanguage": {
* "apex": ["/full/path/to/apex_file1.cls", "/full/path/to/apex_file2.trigger", ...],
* "runDataPerLanguage": {
* "apex": {
* "filesToScan": ["/full/path/to/apex_file1.cls", "/full/path/to/apex_file2.trigger", ...],
* "minimumTokens": 100
* },
* ...,
* "xml": ["full/path/to/xml_file1.xml", "/full/path/to/xml_file2.xml", ...]
* "xml": {
* "filesToScan": ["full/path/to/xml_file1.xml", "/full/path/to/xml_file2.xml", ...],
* "minimumTokens": 150
* }
* },
* "minimumTokens": 100,
* "skipDuplicateFiles": false
* }
* - {resultsOutputFile} is a JSON file to write CPD results to.
Expand Down
119 changes: 86 additions & 33 deletions ...-engine/pmd-cpd-wrappers/src/test/java/com/salesforce/sfca/cpdwrapper/CpdWrapperTest.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -99,43 +99,80 @@ void whenCallingMainWithRunAndInputFileThatDoesNotContainValidJson_thenError(@Te
}

@Test
void whenCallingRunWithMissingField_filesToScanPerLanguage_thenError(@TempDir Path tempDir) throws Exception {
void whenCallingRunWithMissingField_runDataPerLanguage_thenError(@TempDir Path tempDir) throws Exception {
String inputFileContents = "{" +
" \"minimumTokens\": 100, " +
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);

String[] args = {"run", inputFile, "/does/not/matter"};
RuntimeException thrown = assertThrows(RuntimeException.class, () -> callCpdWrapper(args));
assertThat(thrown.getMessage(), is(
"Error while attempting to invoke CpdRunner.run: The \"filesToScanPerLanguage\" field was not set."));
"Error while attempting to invoke CpdRunner.run: The \"runDataPerLanguage\" field was not set."));
}

@Test
void whenCallingRunWithMissingField_filesToScan_thenError(@TempDir Path tempDir) throws Exception {
String inputFileContents = "{" +
" \"runDataPerLanguage\": {" +
" \"apex\": {" +
" \"minimumTokens\": 100 " +
" }" +
" }," +
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);

String[] args = {"run", inputFile, "/does/not/matter"};
RuntimeException thrown = assertThrows(RuntimeException.class, () -> callCpdWrapper(args));
assertThat(thrown.getMessage(), is(
"Error while attempting to invoke CpdRunner.run: The \"filesToScan\" field was missing or empty for language: apex"));
}

@Test
void whenCallingRunWithEmptyArrayFor_filesToScan_thenError(@TempDir Path tempDir) throws Exception {
String inputFileContents = "{" +
" \"runDataPerLanguage\": {" +
" \"apex\": {" +
" \"filesToScan\": []," +
" \"minimumTokens\": 100 " +
" }" +
" }," +
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);

String[] args = {"run", inputFile, "/does/not/matter"};
RuntimeException thrown = assertThrows(RuntimeException.class, () -> callCpdWrapper(args));
assertThat(thrown.getMessage(), is(
"Error while attempting to invoke CpdRunner.run: The \"filesToScan\" field was missing or empty for language: apex"));
}

@Test
void whenCallingRunWithZeroLanguages_thenError(@TempDir Path tempDir) throws Exception {
String inputFileContents = "{" +
" \"filesToScanPerLanguage\": {" +
" \"runDataPerLanguage\": {" +
" }," +
" \"minimumTokens\": 120," +
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);

String[] args = {"run", inputFile, "/does/not/matter"};
RuntimeException thrown = assertThrows(RuntimeException.class, () -> callCpdWrapper(args));
assertThat(thrown.getMessage(), is(
"Error while attempting to invoke CpdRunner.run: The \"filesToScanPerLanguage\" field was found to be empty."));
"Error while attempting to invoke CpdRunner.run: The \"runDataPerLanguage\" field didn't have any languages listed."));
}

@Test
void whenCallingRunWithInvalidLanguage_thenError(@TempDir Path tempDir) throws Exception {
String dummyFile = createTempFile(tempDir, "dummy", "");
String inputFileContents = "{" +
" \"filesToScanPerLanguage\": {" +
" \"unknownLanguage\": [\"" + makePathJsonSafe(dummyFile) + "\"]" +
" \"runDataPerLanguage\": {" +
" \"unknownLanguage\": {" +
" \"filesToScan\": [\"" + makePathJsonSafe(dummyFile) + "\"]," +
" \"minimumTokens\": 120" +
" }" +
" }," +
" \"minimumTokens\": 120," +
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);
Expand All @@ -150,8 +187,10 @@ void whenCallingRunWithInvalidLanguage_thenError(@TempDir Path tempDir) throws E
void whenCallingRunWithMissingField_minimumTokens_thenError(@TempDir Path tempDir) throws Exception {
String dummyApexFile = createTempFile(tempDir, "dummy.cls", "");
String inputFileContents = "{" +
" \"filesToScanPerLanguage\": {" +
" \"apex\": [\"" + makePathJsonSafe(dummyApexFile) + "\"]" +
" \"runDataPerLanguage\": {" +
" \"apex\": {" +
" \"filesToScan\": [\"" + makePathJsonSafe(dummyApexFile) + "\"]" +
" }" +
" }," +
" \"skipDuplicateFiles\": false " +
"}";
Expand All @@ -160,35 +199,39 @@ void whenCallingRunWithMissingField_minimumTokens_thenError(@TempDir Path tempDi
String[] args = {"run", inputFile, "/does/not/matter"};
RuntimeException thrown = assertThrows(RuntimeException.class, () -> callCpdWrapper(args));
assertThat(thrown.getMessage(), is(
"Error while attempting to invoke CpdRunner.run: The \"minimumTokens\" field was not set to a positive number."));
"Error while attempting to invoke CpdRunner.run: The \"minimumTokens\" field was not set to a positive number for language: apex"));
}

@Test
void whenCallingRunWithNegativeMinimumTokensValue_thenError(@TempDir Path tempDir) throws Exception {
String dummyApexFile = createTempFile(tempDir, "dummy.cls", "");
String inputFileContents = "{" +
" \"filesToScanPerLanguage\": {" +
" \"apex\": [\"" + makePathJsonSafe(dummyApexFile) + "\"]" +
" \"runDataPerLanguage\": {" +
" \"apex\": {" +
" \"filesToScan\": [\"" + makePathJsonSafe(dummyApexFile) + "\"]," +
" \"minimumTokens\": -1" +
" }" +
" }," +
" \"minimumTokens\": -1," +
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);

String[] args = {"run", inputFile, "/does/not/matter"};
RuntimeException thrown = assertThrows(RuntimeException.class, () -> callCpdWrapper(args));
assertThat(thrown.getMessage(), is(
"Error while attempting to invoke CpdRunner.run: The \"minimumTokens\" field was not set to a positive number."));
"Error while attempting to invoke CpdRunner.run: The \"minimumTokens\" field was not set to a positive number for language: apex"));
}

@Test
void whenCallingRunWithFileToScanThatDoesNotExist_thenExceptionIsForwardedAsProcessingErrorWithTerminatingExceptionMarker(@TempDir Path tempDir) throws Exception {
String doesNotExist = tempDir.resolve("doesNotExist.cls").toAbsolutePath().toString();
String inputFileContents = "{" +
" \"filesToScanPerLanguage\": {" +
" \"apex\": [\"" + makePathJsonSafe(doesNotExist) + "\"]" +
" \"runDataPerLanguage\": {" +
" \"apex\": {" +
" \"filesToScan\": [\"" + makePathJsonSafe(doesNotExist) + "\"]," +
" \"minimumTokens\": 100" +
" }" +
" }," +
" \"minimumTokens\": 100," +
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);
Expand All @@ -212,11 +255,16 @@ void whenCallingRunWithValidFilesThatHaveDuplicates_thenJsonOutputShouldContainR
String jsFile2 = createTempFile(tempDir, "jsFile2.js", SAMPLE_JS_2);

String inputFileContents = "{" +
" \"filesToScanPerLanguage\": {" +
" \"apex\": [\"" + makePathJsonSafe(apexFile1) + "\", \"" + makePathJsonSafe(apexFile2) + "\"]," +
" \"ecmascript\": [\"" + makePathJsonSafe(jsFile1) + "\", \"" + makePathJsonSafe(jsFile2) + "\"]" +
" \"runDataPerLanguage\": {" +
" \"apex\": {" +
" \"filesToScan\": [\"" + makePathJsonSafe(apexFile1) + "\", \"" + makePathJsonSafe(apexFile2) + "\"]," +
" \"minimumTokens\": 5" +
" }," +
" \"ecmascript\": {" +
" \"filesToScan\": [\"" + makePathJsonSafe(jsFile1) + "\", \"" + makePathJsonSafe(jsFile2) + "\"]," +
" \"minimumTokens\": 13" +
" }" +
" }," +
" \"minimumTokens\": 5," +
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);
Expand Down Expand Up @@ -344,10 +392,12 @@ void whenCallingRunWithValidFilesHaveZeroDuplicatesSinceMinTokensIsHigh_thenJson
String apexFile2 = createTempFile(tempDir, "ApexClass2.cls", SAMPLE_APEX_2);

String inputFileContents = "{" +
" \"filesToScanPerLanguage\": {" +
" \"apex\": [\"" + makePathJsonSafe(apexFile1) + "\", \"" + makePathJsonSafe(apexFile2) + "\"]" +
" \"runDataPerLanguage\": {" +
" \"apex\": {" +
" \"filesToScan\": [\"" + makePathJsonSafe(apexFile1) + "\", \"" + makePathJsonSafe(apexFile2) + "\"]," +
" \"minimumTokens\": 500" +
" }" +
" }," +
" \"minimumTokens\": 500," + // This is why there are no dups found
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);
Expand All @@ -370,11 +420,12 @@ void whenCallingRunWithTwoIdenticalFilesButSkipDuplicateFilesIsFalse_thenJsonOut
String apexFileInSubFolder = createTempFile(subFolder, "ApexClass1.cls", SAMPLE_APEX_1);

String inputFileContents = "{" +
" \"filesToScanPerLanguage\": {" +
" \"apex\": [\"" + makePathJsonSafe(apexFileInParentFolder) + "\", \"" + makePathJsonSafe(apexFileInSubFolder) + "\"]," +
" \"xml\": []" + // Edge case - checking also that this doesn't blow up anything
" \"runDataPerLanguage\": {" +
" \"apex\": {" +
" \"filesToScan\": [\"" + makePathJsonSafe(apexFileInParentFolder) + "\", \"" + makePathJsonSafe(apexFileInSubFolder) + "\"]," +
" \"minimumTokens\": 15" +
" }" +
" }," +
" \"minimumTokens\": 15," +
" \"skipDuplicateFiles\": false " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);
Expand Down Expand Up @@ -434,10 +485,12 @@ void whenCallingRunWithTwoIdenticalFilesButSkipDuplicateFilesIsTrue_thenJsonOutp
String apexFileInSubFolder = createTempFile(subFolder, "ApexClass1.cls", SAMPLE_APEX_1);

String inputFileContents = "{" +
" \"filesToScanPerLanguage\": {" +
" \"apex\": [\"" + makePathJsonSafe(apexFileInParentFolder) + "\", \"" + makePathJsonSafe(apexFileInSubFolder) + "\"]" +
" \"runDataPerLanguage\": {" +
" \"apex\": {" +
" \"filesToScan\": [\"" + makePathJsonSafe(apexFileInParentFolder) + "\", \"" + makePathJsonSafe(apexFileInSubFolder) + "\"]," +
" \"minimumTokens\": 15" +
" }" +
" }," +
" \"minimumTokens\": 15," +
" \"skipDuplicateFiles\": true " +
"}";
String inputFile = createTempFile(tempDir, "inputFile.json", inputFileContents);
Expand Down
Loading
Loading