Skip to content

Commit 0fd8818

Browse files
nahsranahsra
authored
Single codemod execution performance improvements (#468)
* Only load codemod resources if needed (a codemod for the provider is active) * Cache AppScan location data
1 parent d9b49a0 commit 0fd8818

File tree

10 files changed

+140
-58
lines changed

10 files changed

+140
-58
lines changed

‎framework/codemodder-base/src/main/java/io/codemodder/CLI.java‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -366,10 +366,15 @@ public Integer call() throws IOException {
366366
log.debug("excluding paths: {}", pathExcludes);
367367

368368
// get all files that match
369+
log.debug("Listing source directories");
369370
List<SourceDirectory> sourceDirectories =
370371
sourceDirectoryLister.listJavaSourceDirectories(List.of(projectDirectory));
372+
373+
log.debug("Listing files");
371374
List<Path> filePaths = fileFinder.findFiles(projectPath, includesExcludes);
372375

376+
log.debug("Creating codemod regulator");
377+
373378
// get codemod includes/excludes
374379
final CodemodRegulator regulator;
375380
if (codemodIncludes != null && codemodExcludes != null) {
@@ -386,10 +391,13 @@ public Integer call() throws IOException {
386391
}
387392

388393
// create the loader
394+
log.debug("Loading input files");
389395
CodeDirectory codeDirectory = new DefaultCodeDirectory(projectPath);
390396
List<Path> sarifFiles = convertToPaths(sarifs);
391397
List<Path> sonarIssuesJsonFiles = convertToPaths(sonarIssuesJsonFilePaths);
392398
List<Path> sonarHotspotJsonFiles = convertToPaths(sonarHotspotsJsonFilePaths);
399+
400+
log.debug("Parsing SARIFs");
393401
Map<String, List<RuleSarif>> pathSarifMap =
394402
SarifParser.create().parseIntoMap(sarifFiles, codeDirectory);
395403
List<ParameterArgument> codemodParameters =

‎framework/codemodder-base/src/main/java/io/codemodder/CodemodLoader.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
import java.util.regex.Pattern;
1414
import javax.inject.Inject;
1515
import org.jetbrains.annotations.VisibleForTesting;
16+
import org.slf4j.Logger;
17+
import org.slf4j.LoggerFactory;
1618

1719
/** This type is responsible for loading codemods and the surrounding subsystem. */
1820
public final class CodemodLoader {
@@ -33,6 +35,8 @@ public CodemodLoader(
3335
final Path defectDojoFindingsJsonFile,
3436
final Path contrastVulnerabilitiesXmlFilePath) {
3537

38+
log.debug("Loading providers");
39+
3640
// get all the providers ready for dependency injection & codemod instantiation
3741
final List<CodemodProvider> providers =
3842
ServiceLoader.load(CodemodProvider.class).stream()
@@ -106,6 +110,7 @@ public CodemodLoader(
106110
wantsSarif.stream()
107111
.flatMap(toolName -> ruleSarifByTool.getOrDefault(toolName, List.of()).stream())
108112
.toList();
113+
log.debug("Loading modules from provider: {}", provider.getClass().getSimpleName());
109114
final Set<AbstractModule> modules =
110115
provider.getModules(
111116
repositoryDir,
@@ -125,7 +130,9 @@ public CodemodLoader(
125130
final List<CodemodIdPair> codemods = new ArrayList<>();
126131

127132
// validate and instantiate the codemods
133+
log.debug("Instantiating codemods");
128134
final Injector injector = Guice.createInjector(allModules);
135+
log.debug("Codemods instantiated");
129136
final Set<String> codemodIds = new HashSet<>();
130137
for (final Class<? extends CodeChanger> type : orderedCodemodTypes) {
131138
final Codemod codemodAnnotation = type.getAnnotation(Codemod.class);
@@ -164,6 +171,8 @@ static boolean isValidCodemodId(final String codemodId) {
164171
return codemodIdPattern.matcher(codemodId).matches();
165172
}
166173

174+
private static final Logger log = LoggerFactory.getLogger(CodemodLoader.class);
175+
167176
private static final Pattern codemodIdPattern =
168177
Pattern.compile("^([A-Za-z0-9]+):(([A-Za-z0-9]+)/)+([A-Za-z0-9\\-\\.]+)$");
169178
}

‎framework/codemodder-base/src/main/java/io/codemodder/DefaultSarifParser.java‎

Lines changed: 24 additions & 14 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@
55
import com.contrastsecurity.sarif.SarifSchema210;
66
import com.fasterxml.jackson.databind.ObjectMapper;
77
import java.io.IOException;
8+
import java.io.InputStream;
89
import java.nio.file.Files;
910
import java.nio.file.Path;
1011
import java.util.*;
@@ -17,10 +18,14 @@ final class DefaultSarifParser implements SarifParser {
1718

1819
private Optional<SarifSchema210> readSarifFile(final Path sarifFile) {
1920
try {
20-
return Optional.of(
21-
new ObjectMapper().readValue(Files.newInputStream(sarifFile), SarifSchema210.class));
21+
log.debug("Reading input file: {}", sarifFile);
22+
InputStream stream = Files.newInputStream(sarifFile);
23+
log.debug("Parsing to SARIF input files");
24+
SarifSchema210 sarif = new ObjectMapper().readValue(stream, SarifSchema210.class);
25+
log.debug("Parsed SARIF input files");
26+
return Optional.of(sarif);
2227
} catch (final IOException e) {
23-
LOG.error("Problem deserializing SARIF file: {}", sarifFile, e);
28+
log.error("Problem deserializing SARIF file: {}", sarifFile, e);
2429
return Optional.empty();
2530
}
2631
}
@@ -33,14 +38,15 @@ private Optional<Map.Entry<String, RuleSarif>> tryToBuild(
3338
final CodeDirectory codeDirectory,
3439
final List<RuleSarifFactory> factories) {
3540
for (final var factory : factories) {
41+
log.debug("Building SARIF: {}", factory.getClass().getSimpleName());
3642
final var maybeRuleSarif =
3743
factory.build(toolName, rule.ruleId, rule.messageText, sarif, codeDirectory);
3844
if (maybeRuleSarif.isPresent()) {
3945
return Optional.of(Map.entry(toolName, maybeRuleSarif.get()));
4046
}
4147
}
4248

43-
LOG.info("Found SARIF from unsupported tool: {}", toolName);
49+
log.info("Found SARIF from unsupported tool: {}", toolName);
4450
return Optional.empty();
4551
}
4652

@@ -60,7 +66,7 @@ private RuleDescriptor extractRuleId(final Result result, final Run run) {
6066
if (maybeRule.isPresent()) {
6167
return maybeRule.get();
6268
} else {
63-
LOG.info("Could not find rule id for result.");
69+
log.info("Could not find rule id for result.");
6470
return null;
6571
}
6672
}
@@ -72,10 +78,12 @@ private Stream<Map.Entry<String, RuleSarif>> fromSarif(
7278
final Run run, final SarifSchema210 sarif, final CodeDirectory codeDirectory) {
7379
// driver name
7480
final var toolName = run.getTool().getDriver().getName();
81+
log.debug("Loading SARIF rule factories");
7582
final List<RuleSarifFactory> factories =
7683
ServiceLoader.load(RuleSarifFactory.class).stream()
7784
.map(ServiceLoader.Provider::get)
7885
.toList();
86+
log.debug("Done loading SARIF rule factories");
7987
final var runResults = run.getResults();
8088
final var allResults =
8189
runResults != null
@@ -105,16 +113,18 @@ public Map<String, List<RuleSarif>> parseIntoMap(
105113
.flatMap(
106114
sarif -> sarif.getRuns().stream().flatMap(run -> fromSarif(run, sarif, codeDirectory)))
107115
.forEach(
108-
p ->
109-
map.merge(
110-
p.getKey(),
111-
new ArrayList<>(Collections.singletonList(p.getValue())),
112-
(l1, l2) -> {
113-
l1.add(l2.get(0));
114-
return l1;
115-
}));
116+
p -> {
117+
log.debug("Merging SARIF results");
118+
map.merge(
119+
p.getKey(),
120+
new ArrayList<>(Collections.singletonList(p.getValue())),
121+
(l1, l2) -> {
122+
l1.add(l2.get(0));
123+
return l1;
124+
});
125+
});
116126
return map;
117127
}
118128

119-
private static final Logger LOG = LoggerFactory.getLogger(DefaultSarifParser.class);
129+
private static final Logger log = LoggerFactory.getLogger(DefaultSarifParser.class);
120130
}

‎plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarif.java‎

Lines changed: 12 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -1,58 +1,32 @@
11
package io.codemodder.providers.sarif.appscan;
22

33
import com.contrastsecurity.sarif.*;
4-
import io.codemodder.CodeDirectory;
54
import io.codemodder.RuleSarif;
6-
import java.io.IOException;
7-
import java.io.UncheckedIOException;
85
import java.nio.file.Path;
96
import java.util.*;
7+
import org.slf4j.Logger;
8+
import org.slf4j.LoggerFactory;
109

1110
/** A {@link RuleSarif} for AppScan results. */
1211
final class AppScanRuleSarif implements RuleSarif {
1312

1413
private final SarifSchema210 sarif;
1514
private final String messageText;
1615
private final Map<Path, List<Result>> resultsCache;
17-
private final List<String> locations;
18-
19-
/** A map of a AppScan SARIF "location" URIs mapped to their respective file paths. */
20-
private final Map<Path, Set<Integer>> artifactLocationIndices;
16+
private final AppScanSarifLocationData sarifLocationData;
2117

2218
/**
2319
* Creates an {@link AppScanRuleSarif} that has already done the work of mapping AppScan SARIF
2420
* locations, which are strange combinations of class name and file path, into predictable paths.
2521
*/
2622
AppScanRuleSarif(
27-
final String messageText, final SarifSchema210 sarif, final CodeDirectory codeDirectory) {
23+
final String messageText,
24+
final SarifSchema210 sarif,
25+
final AppScanSarifLocationData sarifLocationData) {
2826
this.sarif = Objects.requireNonNull(sarif);
2927
this.messageText = Objects.requireNonNull(messageText);
3028
this.resultsCache = new HashMap<>();
31-
this.locations =
32-
sarif.getRuns().get(0).getArtifacts().stream()
33-
.map(Artifact::getLocation)
34-
.map(ArtifactLocation::getUri)
35-
.map(u -> u.substring(8)) // trim the file:/// of all results
36-
.toList();
37-
Map<Path, Set<Integer>> artifactLocationIndicesMap = new HashMap<>();
38-
39-
for (int i = 0; i < locations.size(); i++) {
40-
final Integer index = i;
41-
String path = locations.get(i);
42-
path = path.replace('\\', '/');
43-
// we have a real but partial path, now we have to find it in the repository
44-
Optional<Path> existingRealPath;
45-
try {
46-
existingRealPath = codeDirectory.findFilesWithTrailingPath(path);
47-
} catch (IOException e) {
48-
throw new UncheckedIOException(e);
49-
}
50-
51-
// add to the map if we found a matching file
52-
existingRealPath.ifPresent(
53-
p -> artifactLocationIndicesMap.computeIfAbsent(p, k -> new HashSet<>()).add(index));
54-
}
55-
this.artifactLocationIndices = Map.copyOf(artifactLocationIndicesMap);
29+
this.sarifLocationData = Objects.requireNonNull(sarifLocationData);
5630
}
5731

5832
@Override
@@ -71,6 +45,9 @@ public List<Region> getRegionsFromResultsByRule(final Path path) {
7145
*/
7246
@Override
7347
public List<Result> getResultsByLocationPath(final Path path) {
48+
49+
Map<Path, Set<Integer>> artifactLocationIndices =
50+
sarifLocationData.getArtifactLocationIndices();
7451
return resultsCache.computeIfAbsent(
7552
path,
7653
p ->
@@ -117,4 +94,6 @@ public String getRule() {
11794
}
11895

11996
static final String toolName = "HCL AppScan Static Analyzer";
97+
98+
private static final Logger log = LoggerFactory.getLogger(AppScanRuleSarif.class);
12099
}

‎plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanRuleSarifFactory.java‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,11 +4,18 @@
44
import io.codemodder.CodeDirectory;
55
import io.codemodder.RuleSarif;
66
import io.codemodder.RuleSarifFactory;
7-
import java.util.Optional;
7+
import java.util.*;
88

99
/** A factory for building {@link AppScanRuleSarif}s. */
1010
public final class AppScanRuleSarifFactory implements RuleSarifFactory {
1111

12+
/** A map of a AppScan SARIF "location" URIs mapped to their respective file paths. */
13+
private final Map<SarifSchema210, AppScanSarifLocationData> sarifLocationDataCache;
14+
15+
public AppScanRuleSarifFactory() {
16+
this.sarifLocationDataCache = new HashMap<>();
17+
}
18+
1219
@Override
1320
public Optional<RuleSarif> build(
1421
final String toolName,
@@ -17,7 +24,12 @@ public Optional<RuleSarif> build(
1724
final SarifSchema210 sarif,
1825
final CodeDirectory codeDirectory) {
1926
if (AppScanRuleSarif.toolName.equals(toolName)) {
20-
return Optional.of(new AppScanRuleSarif(messageText, sarif, codeDirectory));
27+
AppScanSarifLocationData sarifLocationData = sarifLocationDataCache.get(sarif);
28+
if (sarifLocationData == null) {
29+
sarifLocationData = new AppScanSarifLocationData(sarif, codeDirectory);
30+
sarifLocationDataCache.put(sarif, sarifLocationData);
31+
}
32+
return Optional.of(new AppScanRuleSarif(messageText, sarif, sarifLocationData));
2133
}
2234
return Optional.empty();
2335
}

‎plugins/codemodder-plugin-appscan/src/main/java/io/codemodder/providers/sarif/appscan/AppScanSarifLocationData.java‎

Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
package io.codemodder.providers.sarif.appscan;
2+
3+
import com.contrastsecurity.sarif.Artifact;
4+
import com.contrastsecurity.sarif.ArtifactLocation;
5+
import com.contrastsecurity.sarif.SarifSchema210;
6+
import io.codemodder.CodeDirectory;
7+
import java.io.IOException;
8+
import java.io.UncheckedIOException;
9+
import java.nio.file.Path;
10+
import java.util.*;
11+
import org.slf4j.Logger;
12+
import org.slf4j.LoggerFactory;
13+
14+
/** Holds the locations of the artifacts in the SARIF file. */
15+
final class AppScanSarifLocationData {
16+
17+
private final Map<Path, Set<Integer>> artifactLocationIndices;
18+
19+
AppScanSarifLocationData(final SarifSchema210 sarif, final CodeDirectory codeDirectory) {
20+
log.debug("Cleaning locations");
21+
List<String> locations =
22+
sarif.getRuns().get(0).getArtifacts().stream()
23+
.map(Artifact::getLocation)
24+
.map(ArtifactLocation::getUri)
25+
.map(u -> u.substring(8)) // trim the file:/// of all results
26+
.toList();
27+
28+
Map<Path, Set<Integer>> artifactLocationIndicesMap = new HashMap<>();
29+
30+
log.debug("Calculating locations in project dir");
31+
for (int i = 0; i < locations.size(); i++) {
32+
final Integer index = i;
33+
String path = locations.get(i);
34+
path = path.replace('\\', '/');
35+
// we have a real but partial path, now we have to find it in the repository
36+
Optional<Path> existingRealPath;
37+
try {
38+
existingRealPath = codeDirectory.findFilesWithTrailingPath(path);
39+
} catch (IOException e) {
40+
throw new UncheckedIOException(e);
41+
}
42+
// add to the map if we found a matching file
43+
existingRealPath.ifPresent(
44+
p -> artifactLocationIndicesMap.computeIfAbsent(p, k -> new HashSet<>()).add(index));
45+
}
46+
log.debug("Done calculating locations");
47+
this.artifactLocationIndices = Map.copyOf(artifactLocationIndicesMap);
48+
}
49+
50+
Map<Path, Set<Integer>> getArtifactLocationIndices() {
51+
return artifactLocationIndices;
52+
}
53+
54+
private static final Logger log = LoggerFactory.getLogger(AppScanSarifLocationData.class);
55+
}

‎plugins/codemodder-plugin-appscan/src/test/java/io/codemodder/providers/sarif/appscan/AppScanModuleTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -21,7 +21,7 @@
2121
final class AppScanModuleTest {
2222

2323
@Codemod(
24-
id = "appscan-test:java/finds-stuff",
24+
id = "appscan:java/finds-stuff",
2525
importance = Importance.LOW,
2626
reviewGuidance = ReviewGuidance.MERGE_AFTER_CURSORY_REVIEW)
2727
static class AppScanSarifTestCodemod implements CodeChanger {

‎plugins/codemodder-plugin-pmd/src/main/java/io/codemodder/providers/sarif/pmd/PmdModule.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,7 @@
33
import com.contrastsecurity.sarif.Result;
44
import com.contrastsecurity.sarif.SarifSchema210;
55
import com.google.inject.AbstractModule;
6-
import io.codemodder.CodeChanger;
7-
import io.codemodder.LazyLoadingRuleSarif;
8-
import io.codemodder.RuleSarif;
6+
import io.codemodder.*;
97
import io.github.classgraph.*;
108
import java.lang.reflect.Executable;
119
import java.lang.reflect.Parameter;
@@ -83,6 +81,11 @@ protected void configure() {
8381
}
8482
}
8583

84+
if (scanTargets.isEmpty()) {
85+
LOG.trace("No @PmdScan annotations found, skipping");
86+
return;
87+
}
88+
8689
SarifSchema210 allRulesBatchedRun =
8790
pmdRunner.run(
8891
scanTargets.stream().map(PmdScanTarget::pmdScan).map(PmdScan::ruleId).toList(),

‎plugins/codemodder-plugin-pmd/src/test/java/io/codemodder/providers/sarif/pmd/PmdModuleTest.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ public abstract class MultipleDeclarations {
4646
}
4747

4848
@Codemod(
49-
id = "pmd-test:java/my-pmd-codemod",
49+
id = "pmd:java/my-pmd-codemod",
5050
importance = Importance.HIGH,
5151
reviewGuidance = ReviewGuidance.MERGE_AFTER_CURSORY_REVIEW)
5252
static class UsesPmdCodemod extends SarifPluginJavaParserChanger<VariableDeclarator> {

0 commit comments

Comments
 (0)