Skip to content

Commit d118cbf

Browse files
breskebybreskeby
authored
[Build] Make ThirdPartyAudit task uptodate more effective (#118723)
We should basically be able to skip the third party dependency audit tasks if no third party dependency has changed. Filtering out the project dependencies allows way better uptodate and caching behaviour. We are only interested in thirdparty libs anyhow in this context. A positive side effect of the reduced class path is a quicker execution of the task
1 parent 3f1fed0 commit d118cbf

File tree

3 files changed

+41
-19
lines changed

3 files changed

+41
-19
lines changed

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditPrecommitPlugin.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -16,12 +16,14 @@
1616
import org.gradle.api.Task;
1717
import org.gradle.api.artifacts.Configuration;
1818
import org.gradle.api.artifacts.component.ModuleComponentIdentifier;
19+
import org.gradle.api.file.FileCollection;
1920
import org.gradle.api.tasks.TaskProvider;
2021

2122
import java.io.File;
2223
import java.nio.file.Path;
2324

2425
import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createFileCollectionFromNonTransitiveArtifactsView;
26+
import static org.elasticsearch.gradle.internal.util.DependenciesUtils.thirdPartyDependenciesView;
2527
import static org.elasticsearch.gradle.internal.util.ParamsUtils.loadBuildParams;
2628

2729
public class ThirdPartyAuditPrecommitPlugin extends PrecommitPlugin {
@@ -47,7 +49,6 @@ public TaskProvider<? extends Task> createTask(Project project) {
4749
project.getDependencies().add(JDK_JAR_HELL_CONFIG_NAME, elasticsearchCoreProject);
4850
}
4951
}
50-
5152
TaskProvider<ExportElasticsearchBuildResourcesTask> resourcesTask = project.getTasks()
5253
.register("thirdPartyAuditResources", ExportElasticsearchBuildResourcesTask.class);
5354
Path resourcesDir = project.getBuildDir().toPath().resolve("third-party-audit-config");
@@ -59,9 +60,11 @@ public TaskProvider<? extends Task> createTask(Project project) {
5960
// usually only one task is created. but this construct makes our integTests easier to setup
6061
project.getTasks().withType(ThirdPartyAuditTask.class).configureEach(t -> {
6162
Configuration runtimeConfiguration = project.getConfigurations().getByName("runtimeClasspath");
63+
FileCollection runtimeThirdParty = thirdPartyDependenciesView(runtimeConfiguration);
6264
Configuration compileOnly = project.getConfigurations()
6365
.getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
64-
t.setClasspath(runtimeConfiguration.plus(compileOnly));
66+
FileCollection compileOnlyThirdParty = thirdPartyDependenciesView(compileOnly);
67+
t.getThirdPartyClasspath().from(runtimeThirdParty, compileOnlyThirdParty);
6568
t.getJarsToScan()
6669
.from(
6770
createFileCollectionFromNonTransitiveArtifactsView(
@@ -78,7 +81,7 @@ public TaskProvider<? extends Task> createTask(Project project) {
7881
t.getJavaHome().set(buildParams.flatMap(params -> params.getRuntimeJavaHome()).map(File::getPath));
7982
t.setSignatureFile(resourcesDir.resolve("forbidden/third-party-audit.txt").toFile());
8083
t.getJdkJarHellClasspath().from(jdkJarHellConfig);
81-
t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnly));
84+
t.getForbiddenAPIsClasspath().from(project.getConfigurations().getByName("forbiddenApisCliJar").plus(compileOnlyThirdParty));
8285
});
8386
return audit;
8487
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/ThirdPartyAuditTask.java‎

Lines changed: 3 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717
import org.gradle.api.JavaVersion;
1818
import org.gradle.api.file.ArchiveOperations;
1919
import org.gradle.api.file.ConfigurableFileCollection;
20-
import org.gradle.api.file.FileCollection;
2120
import org.gradle.api.file.FileSystemOperations;
2221
import org.gradle.api.file.FileTree;
2322
import org.gradle.api.file.ProjectLayout;
@@ -96,8 +95,6 @@ public abstract class ThirdPartyAuditTask extends DefaultTask {
9695

9796
private final ProjectLayout projectLayout;
9897

99-
private FileCollection classpath;
100-
10198
@Inject
10299
public ThirdPartyAuditTask(
103100
ArchiveOperations archiveOperations,
@@ -198,9 +195,7 @@ public Set<String> getMissingClassExcludes() {
198195
public abstract Property<JavaVersion> getRuntimeJavaVersion();
199196

200197
@Classpath
201-
public FileCollection getClasspath() {
202-
return classpath;
203-
}
198+
public abstract ConfigurableFileCollection getThirdPartyClasspath();
204199

205200
@TaskAction
206201
public void runThirdPartyAudit() throws IOException {
@@ -345,7 +340,7 @@ private String runForbiddenAPIsCli() throws IOException {
345340
if (javaHome.isPresent()) {
346341
spec.setExecutable(javaHome.get() + "/bin/java");
347342
}
348-
spec.classpath(getForbiddenAPIsClasspath(), classpath);
343+
spec.classpath(getForbiddenAPIsClasspath(), getThirdPartyClasspath());
349344
// Enable explicitly for each release as appropriate. Just JDK 20/21/22/23 for now, and just the vector module.
350345
if (isJavaVersion(VERSION_20) || isJavaVersion(VERSION_21) || isJavaVersion(VERSION_22) || isJavaVersion(VERSION_23)) {
351346
spec.jvmArgs("--add-modules", "jdk.incubator.vector");
@@ -383,7 +378,7 @@ private boolean isJavaVersion(JavaVersion version) {
383378
private Set<String> runJdkJarHellCheck() throws IOException {
384379
ByteArrayOutputStream standardOut = new ByteArrayOutputStream();
385380
ExecResult execResult = execOperations.javaexec(spec -> {
386-
spec.classpath(getJdkJarHellClasspath(), classpath);
381+
spec.classpath(getJdkJarHellClasspath(), getThirdPartyClasspath());
387382
spec.getMainClass().set(JDK_JAR_HELL_MAIN_CLASS);
388383
spec.args(getJarExpandDir());
389384
spec.setIgnoreExitValue(true);
@@ -402,8 +397,4 @@ private Set<String> runJdkJarHellCheck() throws IOException {
402397
return new TreeSet<>(Arrays.asList(jdkJarHellCheckList.split("\\r?\\n")));
403398
}
404399

405-
public void setClasspath(FileCollection classpath) {
406-
this.classpath = classpath;
407-
}
408-
409400
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java‎

Lines changed: 32 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -9,12 +9,16 @@
99

1010
package org.elasticsearch.gradle.internal.util;
1111

12+
import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin;
13+
1214
import org.gradle.api.artifacts.Configuration;
1315
import org.gradle.api.artifacts.ResolvableDependencies;
1416
import org.gradle.api.artifacts.component.ComponentIdentifier;
17+
import org.gradle.api.artifacts.component.ProjectComponentIdentifier;
1518
import org.gradle.api.artifacts.result.ResolvedComponentResult;
1619
import org.gradle.api.artifacts.result.ResolvedDependencyResult;
1720
import org.gradle.api.file.FileCollection;
21+
import org.gradle.api.provider.Provider;
1822
import org.gradle.api.specs.AndSpec;
1923
import org.gradle.api.specs.Spec;
2024

@@ -29,7 +33,7 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
2933
) {
3034
ResolvableDependencies incoming = configuration.getIncoming();
3135
return incoming.artifactView(viewConfiguration -> {
32-
Set<ComponentIdentifier> firstLevelDependencyComponents = incoming.getResolutionResult()
36+
Provider<Set<ComponentIdentifier>> firstLevelDependencyComponents = incoming.getResolutionResult()
3337
.getRootComponent()
3438
.map(
3539
rootComponent -> rootComponent.getDependencies()
@@ -39,12 +43,36 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
3943
.filter(dependency -> dependency.getSelected() instanceof ResolvedComponentResult)
4044
.map(dependency -> dependency.getSelected().getId())
4145
.collect(Collectors.toSet())
42-
)
43-
.get();
46+
);
4447
viewConfiguration.componentFilter(
45-
new AndSpec<>(identifier -> firstLevelDependencyComponents.contains(identifier), componentFilter)
48+
new AndSpec<>(identifier -> firstLevelDependencyComponents.get().contains(identifier), componentFilter)
4649
);
4750
}).getFiles();
4851
}
4952

53+
/**
54+
* This method gives us an artifact view of a configuration that filters out all
55+
* project dependencies that are not shadowed jars.
56+
* Basically a thirdparty only view of the dependency tree.
57+
*/
58+
public static FileCollection thirdPartyDependenciesView(Configuration configuration) {
59+
ResolvableDependencies incoming = configuration.getIncoming();
60+
return incoming.artifactView(v -> {
61+
// resolve componentIdentifier for all shadowed project dependencies
62+
Provider<Set<ComponentIdentifier>> shadowedDependencies = incoming.getResolutionResult()
63+
.getRootComponent()
64+
.map(
65+
root -> root.getDependencies()
66+
.stream()
67+
.filter(dep -> dep instanceof ResolvedDependencyResult)
68+
.map(dep -> (ResolvedDependencyResult) dep)
69+
.filter(dep -> dep.getResolvedVariant().getDisplayName() == ShadowBasePlugin.COMPONENT_NAME)
70+
.filter(dep -> dep.getSelected() instanceof ResolvedComponentResult)
71+
.map(dep -> dep.getSelected().getId())
72+
.collect(Collectors.toSet())
73+
);
74+
// filter out project dependencies if they are not a shadowed dependency
75+
v.componentFilter(i -> (i instanceof ProjectComponentIdentifier == false || shadowedDependencies.get().contains(i)));
76+
}).getFiles();
77+
}
5078
}

0 commit comments

Comments
 (0)