Skip to content

Commit f4f55d7

Browse files
breskebybreskeby
authored
[8.18] Fix dependency reporting (elastic#136209) (elastic#136320)
* Fix dependency reporting (elastic#136209) This has been broken since 9.0.3. likely due to some swallowed api change in Gradle. This fixes and adds test coverage to dependency reporting used in DRA artifacts building Fix interdependency to Licenses task (cherry picked from commit f74406d) # Conflicts: # build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java # modules/repository-s3/build.gradle # plugins/discovery-ec2/build.gradle # x-pack/plugin/core/build.gradle # x-pack/plugin/identity-provider/build.gradle * Fix merge issue * fix merge conflict
1 parent 4601bf1 commit f4f55d7

File tree

17 files changed

+141
-63
lines changed

17 files changed

+141
-63
lines changed

‎build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy‎

Lines changed: 24 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -151,7 +151,11 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
151151
tasks.named('checkstyleMain').configure { enabled = false }
152152
tasks.named('loggerUsageCheck').configure { enabled = false }
153153
// tested elsewhere
154-
tasks.named('thirdPartyAudit').configure { enabled = false }
154+
tasks.named('thirdPartyAudit').configure {
155+
getRuntimeJavaVersion().set(JavaVersion.VERSION_21)
156+
getTargetCompatibility().set(JavaVersion.VERSION_21)
157+
enabled = false
158+
}
155159
"""
156160
when:
157161
def result = gradleRunner("check").build()
@@ -167,6 +171,25 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
167171
result.task(":loggerUsageCheck").outcome == TaskOutcome.SKIPPED
168172
}
169173

174+
def "can generate dependency infos file"() {
175+
given:
176+
repository.generateJar("junit", "junit", "4.12", 'org.acme.JunitMock')
177+
repository.configureBuild(buildFile)
178+
file("licenses/junit-4.12.jar.sha1").text = "2973d150c0dc1fefe998f834810d68f278ea58ec"
179+
file("licenses/junit-LICENSE.txt").text = EXAMPLE_LICENSE
180+
file("licenses/junit-NOTICE.txt").text = "mock notice"
181+
buildFile << """
182+
dependencies {
183+
api "junit:junit:4.12"
184+
}
185+
"""
186+
when:
187+
def result = gradleRunner("dependenciesInfo").build()
188+
then:
189+
result.task(":dependenciesInfo").outcome == TaskOutcome.SUCCESS
190+
file("build/reports/dependencies/dependencies.csv").text == "junit:junit,4.12,https://repo1.maven.org/maven2/junit/junit/4.12,BSD-3-Clause,\n"
191+
}
192+
170193
def assertValidJar(File jar) {
171194
try (ZipFile zipFile = new ZipFile(jar)) {
172195
ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt")

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/AbstractDependenciesTask.java‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,44 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the "Elastic License
4+
* 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
5+
* Public License v 1"; you may not use this file except in compliance with, at
6+
* your election, the "Elastic License 2.0", the "GNU Affero General Public
7+
* License v3.0 only", or the "Server Side Public License, v 1".
8+
*/
9+
10+
package org.elasticsearch.gradle.internal;
11+
12+
import org.gradle.api.DefaultTask;
13+
import org.gradle.api.InvalidUserDataException;
14+
import org.gradle.api.provider.MapProperty;
15+
import org.gradle.api.tasks.Input;
16+
import org.gradle.api.tasks.Optional;
17+
18+
import java.util.Map;
19+
20+
public abstract class AbstractDependenciesTask extends DefaultTask {
21+
22+
@Input
23+
@Optional
24+
public abstract MapProperty<String, String> getMappings();
25+
26+
/**
27+
* Add a mapping from a regex pattern for the jar name, to a prefix to find
28+
* the LICENSE and NOTICE file for that jar.
29+
*/
30+
public void mapping(Map<String, String> props) {
31+
String from = props.get("from");
32+
if (from == null) {
33+
throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
34+
}
35+
String to = props.get("to");
36+
if (to == null) {
37+
throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
38+
}
39+
if (props.size() > 2) {
40+
throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
41+
}
42+
getMappings().put(from, to);
43+
}
44+
}

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717
import org.gradle.api.attributes.Usage;
1818
import org.gradle.api.plugins.JavaPlugin;
1919

20+
import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createNonTransitiveArtifactsView;
21+
2022
public class DependenciesInfoPlugin implements Plugin<Project> {
2123

2224
public static String USAGE_ATTRIBUTE = "DependenciesInfo";
@@ -25,16 +27,15 @@ public class DependenciesInfoPlugin implements Plugin<Project> {
2527
public void apply(final Project project) {
2628
project.getPlugins().apply(CompileOnlyResolvePlugin.class);
2729
var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
28-
2930
depsInfo.configure(t -> {
3031
var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
31-
t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
32+
t.getRuntimeArtifacts()
33+
.set(project.getProviders().provider(() -> createNonTransitiveArtifactsView(runtimeConfiguration).getArtifacts()));
3234
t.getClasspath().from(runtimeConfiguration);
3335
var compileOnlyConfiguration = project.getConfigurations()
3436
.getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
3537
t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
3638
t.getClasspath().from(compileOnlyConfiguration);
37-
3839
});
3940
Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
4041
dependenciesInfoFilesConfiguration.setCanBeResolved(false);

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java‎

Lines changed: 12 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -17,19 +17,21 @@
1717
import org.gradle.api.file.ConfigurableFileCollection;
1818
import org.gradle.api.file.DirectoryProperty;
1919
import org.gradle.api.file.ProjectLayout;
20-
import org.gradle.api.internal.ConventionTask;
2120
import org.gradle.api.model.ObjectFactory;
22-
import org.gradle.api.provider.MapProperty;
2321
import org.gradle.api.provider.Property;
2422
import org.gradle.api.provider.Provider;
2523
import org.gradle.api.provider.ProviderFactory;
24+
import org.gradle.api.tasks.CacheableTask;
25+
import org.gradle.api.tasks.Classpath;
2626
import org.gradle.api.tasks.Input;
2727
import org.gradle.api.tasks.InputDirectory;
28-
import org.gradle.api.tasks.InputFiles;
2928
import org.gradle.api.tasks.Internal;
3029
import org.gradle.api.tasks.Optional;
3130
import org.gradle.api.tasks.OutputFile;
31+
import org.gradle.api.tasks.PathSensitive;
32+
import org.gradle.api.tasks.PathSensitivity;
3233
import org.gradle.api.tasks.TaskAction;
34+
import org.gradle.internal.component.external.model.ModuleComponentArtifactIdentifier;
3335

3436
import java.io.File;
3537
import java.io.IOException;
@@ -55,7 +57,8 @@
5557
* <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
5658
* </ul>
5759
*/
58-
public abstract class DependenciesInfoTask extends ConventionTask {
60+
@CacheableTask
61+
public abstract class DependenciesInfoTask extends AbstractDependenciesTask {
5962

6063
@Inject
6164
public abstract ProviderFactory getProviderFactory();
@@ -86,16 +89,17 @@ public Provider<Set<ModuleComponentIdentifier>> getCompileOnlyModules() {
8689
* artifact transforms that might be applied and fail due to missing task dependency to jar
8790
* generating tasks.
8891
* */
89-
@InputFiles
92+
@Classpath
9093
abstract ConfigurableFileCollection getClasspath();
9194

9295
private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers(ArtifactCollection artifacts) {
9396
return getProviderFactory().provider(
9497
() -> artifacts.getArtifacts()
9598
.stream()
9699
.map(r -> r.getId())
97-
.filter(id -> id instanceof ModuleComponentIdentifier)
98-
.map(id -> (ModuleComponentIdentifier) id)
100+
.filter(mcaId -> mcaId instanceof ModuleComponentArtifactIdentifier)
101+
.map(mcaId -> (ModuleComponentArtifactIdentifier) mcaId)
102+
.map(it -> it.getComponentIdentifier())
99103
.collect(Collectors.toSet())
100104
);
101105
}
@@ -111,6 +115,7 @@ private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers
111115
* Directory to read license files
112116
*/
113117
@Optional
118+
@PathSensitive(PathSensitivity.RELATIVE)
114119
@InputDirectory
115120
public File getLicensesDir() {
116121
File asFile = licensesDir.get().getAsFile();
@@ -143,7 +148,6 @@ public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFac
143148

144149
@TaskAction
145150
public void generateDependenciesInfo() throws IOException {
146-
147151
final Set<String> compileOnlyIds = getCompileOnlyModules().map(
148152
set -> set.stream()
149153
.map(id -> id.getModuleIdentifier().getGroup() + ":" + id.getModuleIdentifier().getName() + ":" + id.getVersion())
@@ -166,18 +170,13 @@ public void generateDependenciesInfo() throws IOException {
166170
final String url = createURL(dep.getGroup(), moduleName, dep.getVersion());
167171
final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, moduleName);
168172
getLogger().info("mapped dependency " + dep.getGroup() + ":" + moduleName + " to " + dependencyName + " for license info");
169-
170173
final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
171174
output.append(dep.getGroup() + ":" + moduleName + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
172175
}
173176

174177
Files.write(outputFile.toPath(), output.toString().getBytes("UTF-8"), StandardOpenOption.CREATE);
175178
}
176179

177-
@Input
178-
@Optional
179-
public abstract MapProperty<String, String> getMappings();
180-
181180
/**
182181
* Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>
183182
* based on dependency coordinates.

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java‎

Lines changed: 3 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,9 @@
88
*/
99
package org.elasticsearch.gradle.internal.precommit;
1010

11+
import org.elasticsearch.gradle.internal.AbstractDependenciesTask;
1112
import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer.LicenseInfo;
12-
import org.gradle.api.DefaultTask;
1313
import org.gradle.api.GradleException;
14-
import org.gradle.api.InvalidUserDataException;
1514
import org.gradle.api.artifacts.Configuration;
1615
import org.gradle.api.artifacts.component.ComponentIdentifier;
1716
import org.gradle.api.file.Directory;
@@ -39,7 +38,6 @@
3938
import java.util.Arrays;
4039
import java.util.HashMap;
4140
import java.util.HashSet;
42-
import java.util.LinkedHashMap;
4341
import java.util.LinkedHashSet;
4442
import java.util.List;
4543
import java.util.Map;
@@ -94,7 +92,7 @@
9492
* comply with the license terms.
9593
*/
9694
@CacheableTask
97-
public abstract class DependencyLicensesTask extends DefaultTask {
95+
public abstract class DependencyLicensesTask extends AbstractDependenciesTask {
9896

9997
private final Pattern regex = Pattern.compile("-v?\\d+.*");
10098

@@ -112,11 +110,6 @@ public abstract class DependencyLicensesTask extends DefaultTask {
112110
*/
113111
private final DirectoryProperty licensesDir;
114112

115-
/**
116-
* A map of patterns to prefix, used to find the LICENSE and NOTICE file.
117-
*/
118-
private Map<String, String> mappings = new LinkedHashMap<>();
119-
120113
/**
121114
* Names of dependencies whose shas should not exist.
122115
*/
@@ -128,25 +121,6 @@ public abstract class DependencyLicensesTask extends DefaultTask {
128121
private LinkedHashSet<String> ignoreFiles = new LinkedHashSet<>();
129122
private ProjectLayout projectLayout;
130123

131-
/**
132-
* Add a mapping from a regex pattern for the jar name, to a prefix to find
133-
* the LICENSE and NOTICE file for that jar.
134-
*/
135-
public void mapping(Map<String, String> props) {
136-
String from = props.get("from");
137-
if (from == null) {
138-
throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
139-
}
140-
String to = props.get("to");
141-
if (to == null) {
142-
throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
143-
}
144-
if (props.size() > 2) {
145-
throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
146-
}
147-
mappings.put(from, to);
148-
}
149-
150124
@Inject
151125
public DependencyLicensesTask(ObjectFactory objects, ProjectLayout projectLayout) {
152126
this.projectLayout = projectLayout;
@@ -267,7 +241,7 @@ private void checkDependencies(Map<String, Boolean> licenses, Map<String, Boolea
267241
for (File dependency : dependencies) {
268242
String jarName = dependency.getName();
269243
String depName = regex.matcher(jarName).replaceFirst("");
270-
String dependencyName = getDependencyName(mappings, depName);
244+
String dependencyName = getDependencyName(getMappings().get(), depName);
271245
logger.info("mapped dependency name {} to {} for license/notice check", depName, dependencyName);
272246
checkFile(dependencyName, jarName, licenses, "LICENSE");
273247
checkFile(dependencyName, jarName, notices, "NOTICE");
@@ -321,11 +295,6 @@ public LinkedHashSet<String> getIgnoreFiles() {
321295
return new LinkedHashSet<>(ignoreFiles);
322296
}
323297

324-
@Input
325-
public LinkedHashMap<String, String> getMappings() {
326-
return new LinkedHashMap<>(mappings);
327-
}
328-
329298
/**
330299
* Convencience method for configuring dependencies to be checked and ignoring transitive dependencies for now.
331300
* */

‎build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@
99

1010
package org.elasticsearch.gradle.internal.util;
1111

12+
import org.gradle.api.artifacts.ArtifactView;
1213
import org.gradle.api.artifacts.Configuration;
1314
import org.gradle.api.artifacts.ResolvableDependencies;
1415
import org.gradle.api.artifacts.component.ComponentIdentifier;
@@ -27,6 +28,14 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
2728
Configuration configuration,
2829
Spec<ComponentIdentifier> componentFilter
2930
) {
31+
return createNonTransitiveArtifactsView(configuration, componentFilter).getFiles();
32+
}
33+
34+
public static ArtifactView createNonTransitiveArtifactsView(Configuration configuration) {
35+
return createNonTransitiveArtifactsView(configuration, identifier -> true);
36+
}
37+
38+
public static ArtifactView createNonTransitiveArtifactsView(Configuration configuration, Spec<ComponentIdentifier> componentFilter) {
3039
ResolvableDependencies incoming = configuration.getIncoming();
3140
return incoming.artifactView(viewConfiguration -> {
3241
Set<ComponentIdentifier> firstLevelDependencyComponents = incoming.getResolutionResult()
@@ -44,6 +53,6 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
4453
viewConfiguration.componentFilter(
4554
new AndSpec<>(identifier -> firstLevelDependencyComponents.contains(identifier), componentFilter)
4655
);
47-
}).getFiles();
56+
});
4857
}
4958
}

‎client/sniffer/build.gradle‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717
* under the License.
1818
*/
1919
import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersTask
20+
import org.elasticsearch.gradle.internal.AbstractDependenciesTask
2021

2122
apply plugin: 'elasticsearch.build'
2223
apply plugin: 'elasticsearch.publish'
@@ -67,7 +68,7 @@ tasks.named('forbiddenApisTest').configure {
6768
replaceSignatureFiles 'jdk-signatures'
6869
}
6970

70-
tasks.named("dependencyLicenses").configure {
71+
tasks.withType(AbstractDependenciesTask).configureEach {
7172
mapping from: /http.*/, to: 'httpclient'
7273
mapping from: /commons-.*/, to: 'commons'
7374
}

‎distribution/build.gradle‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,11 @@ tasks.register("generateDependenciesReport", ConcatFilesTask) {
8383
'https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/8/ubi-minimal-8-source.tar.gz'
8484
]
8585
additionalLines << rhelUbiFields.join(',')
86+
doLast {
87+
if(target.text.readLines().size() < 100) {
88+
throw new GradleException("Suspiciously low number of dependencies. Double check.")
89+
}
90+
}
8691
}
8792

8893
/*****************************************************************************

‎modules/ingest-geoip/build.gradle‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -8,6 +8,7 @@
88
*/
99

1010
import org.elasticsearch.gradle.OS
11+
import org.elasticsearch.gradle.internal.AbstractDependenciesTask
1112

1213
apply plugin: 'elasticsearch.internal-yaml-rest-test'
1314
apply plugin: 'elasticsearch.yaml-rest-compat-test'
@@ -78,10 +79,13 @@ tasks.named("forbiddenPatterns").configure {
7879
exclude '**/*.mmdb'
7980
}
8081

81-
tasks.named("dependencyLicenses").configure {
82+
tasks.withType(AbstractDependenciesTask).configureEach {
8283
mapping from: /geoip.*/, to: 'maxmind-geolite2-eula'
8384
mapping from: /maxmind-db.*/, to: 'maxmind-db-reader'
8485
mapping from: /jackson.*/, to: 'jackson'
86+
}
87+
88+
tasks.named("dependencyLicenses").configure {
8589
ignoreFile 'elastic-geoip-database-service-agreement-LICENSE.txt'
8690
}
8791

‎modules/repository-s3/build.gradle‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,8 @@ import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
1010
* your election, the "Elastic License 2.0", the "GNU Affero General Public
1111
* License v3.0 only", or the "Server Side Public License, v 1".
1212
*/
13+
import org.elasticsearch.gradle.internal.AbstractDependenciesTask
14+
1315
apply plugin: 'elasticsearch.internal-yaml-rest-test'
1416
apply plugin: 'elasticsearch.internal-cluster-test'
1517
apply plugin: 'elasticsearch.internal-java-rest-test'
@@ -68,7 +70,7 @@ restResources {
6870
}
6971
}
7072

71-
tasks.named("dependencyLicenses").configure {
73+
tasks.withType(AbstractDependenciesTask).configureEach {
7274
mapping from: /aws-java-sdk-.*/, to: 'aws-java-sdk'
7375
mapping from: /jmespath-java.*/, to: 'aws-java-sdk'
7476
mapping from: /jackson-.*/, to: 'jackson'

0 commit comments

Comments
 (0)