Fix dependency reporting (elastic#136209) (elastic#136316)

This has been broken since 9.0.3. likely due to some swallowed api change in
Gradle. This fixes and adds test coverage to dependency reporting used in DRA
artifacts building

Fix interdependency to Licenses task

(cherry picked from commit f74406d)

Author: breskeby

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy

@@ -151,7 +151,11 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
             tasks.named('checkstyleMain').configure { enabled = false }
             tasks.named('loggerUsageCheck').configure { enabled = false }
             // tested elsewhere
-            tasks.named('thirdPartyAudit').configure { enabled = false }
+            tasks.named('thirdPartyAudit').configure {
+                getRuntimeJavaVersion().set(JavaVersion.VERSION_21)
+                getTargetCompatibility().set(JavaVersion.VERSION_21)
+                enabled = false
+            }
             """
         when:
         def result = gradleRunner("check").build()
@@ -167,6 +171,25 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
         result.task(":loggerUsageCheck").outcome == TaskOutcome.SKIPPED
     }
 
+    def "can generate dependency infos file"() {
+        given:
+        repository.generateJar("junit", "junit", "4.12", 'org.acme.JunitMock')
+        repository.configureBuild(buildFile)
+        file("licenses/junit-4.12.jar.sha1").text = "2973d150c0dc1fefe998f834810d68f278ea58ec"
+        file("licenses/junit-LICENSE.txt").text = EXAMPLE_LICENSE
+        file("licenses/junit-NOTICE.txt").text = "mock notice"
+        buildFile << """
+        dependencies {
+            api "junit:junit:4.12"
+        }
+        """
+        when:
+        def result = gradleRunner("dependenciesInfo").build()
+        then:
+        result.task(":dependenciesInfo").outcome == TaskOutcome.SUCCESS
+        file("build/reports/dependencies/dependencies.csv").text == "junit:junit,4.12,https://repo1.maven.org/maven2/junit/junit/4.12,BSD-3-Clause,\n"
+    }
+
     def assertValidJar(File jar) {
         try (ZipFile zipFile = new ZipFile(jar)) {
             ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt")

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/AbstractDependenciesTask.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.InvalidUserDataException;
+import org.gradle.api.provider.MapProperty;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.Optional;
+
+import java.util.Map;
+
+public abstract class AbstractDependenciesTask extends DefaultTask {
+
+    @Input
+    @Optional
+    public abstract MapProperty<String, String> getMappings();
+
+    /**
+     * Add a mapping from a regex pattern for the jar name, to a prefix to find
+     * the LICENSE and NOTICE file for that jar.
+     */
+    public void mapping(Map<String, String> props) {
+        String from = props.get("from");
+        if (from == null) {
+            throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
+        }
+        String to = props.get("to");
+        if (to == null) {
+            throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
+        }
+        if (props.size() > 2) {
+            throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
+        }
+        getMappings().put(from, to);
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java

@@ -17,6 +17,8 @@
 import org.gradle.api.attributes.Usage;
 import org.gradle.api.plugins.JavaPlugin;
 
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createNonTransitiveArtifactsView;
+
 public class DependenciesInfoPlugin implements Plugin<Project> {
 
     public static String USAGE_ATTRIBUTE = "DependenciesInfo";
@@ -25,16 +27,15 @@ public class DependenciesInfoPlugin implements Plugin<Project> {
     public void apply(final Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
         var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
-
         depsInfo.configure(t -> {
             var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
-            t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
+            t.getRuntimeArtifacts()
+                .set(project.getProviders().provider(() -> createNonTransitiveArtifactsView(runtimeConfiguration).getArtifacts()));
             t.getClasspath().from(runtimeConfiguration);
             var compileOnlyConfiguration = project.getConfigurations()
                 .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
             t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
             t.getClasspath().from(compileOnlyConfiguration);
-
         });
         Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
         dependenciesInfoFilesConfiguration.setCanBeResolved(false);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java

@@ -17,19 +17,21 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.ProjectLayout;
-import org.gradle.api.internal.ConventionTask;
 import org.gradle.api.model.ObjectFactory;
-import org.gradle.api.provider.MapProperty;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.Classpath;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
+import org.gradle.internal.component.external.model.ModuleComponentArtifactIdentifier;
 
 import java.io.File;
 import java.io.IOException;
@@ -55,7 +57,8 @@
  *     <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
  * </ul>
  */
-public abstract class DependenciesInfoTask extends ConventionTask {
+@CacheableTask
+public abstract class DependenciesInfoTask extends AbstractDependenciesTask {
 
     @Inject
     public abstract ProviderFactory getProviderFactory();
@@ -86,16 +89,17 @@ public Provider<Set<ModuleComponentIdentifier>> getCompileOnlyModules() {
      * artifact transforms that might be applied and fail due to missing task dependency to jar
      * generating tasks.
      * */
-    @InputFiles
+    @Classpath
     abstract ConfigurableFileCollection getClasspath();
 
     private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers(ArtifactCollection artifacts) {
         return getProviderFactory().provider(
             () -> artifacts.getArtifacts()
                 .stream()
                 .map(r -> r.getId())
-                .filter(id -> id instanceof ModuleComponentIdentifier)
-                .map(id -> (ModuleComponentIdentifier) id)
+                .filter(mcaId -> mcaId instanceof ModuleComponentArtifactIdentifier)
+                .map(mcaId -> (ModuleComponentArtifactIdentifier) mcaId)
+                .map(it -> it.getComponentIdentifier())
                 .collect(Collectors.toSet())
         );
     }
@@ -111,6 +115,7 @@ private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers
      * Directory to read license files
      */
     @Optional
+    @PathSensitive(PathSensitivity.RELATIVE)
     @InputDirectory
     public File getLicensesDir() {
         File asFile = licensesDir.get().getAsFile();
@@ -143,7 +148,6 @@ public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFac
 
     @TaskAction
     public void generateDependenciesInfo() throws IOException {
-
         final Set<String> compileOnlyIds = getCompileOnlyModules().map(
             set -> set.stream()
                 .map(id -> id.getModuleIdentifier().getGroup() + ":" + id.getModuleIdentifier().getName() + ":" + id.getVersion())
@@ -166,18 +170,13 @@ public void generateDependenciesInfo() throws IOException {
             final String url = createURL(dep.getGroup(), moduleName, dep.getVersion());
             final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, moduleName);
             getLogger().info("mapped dependency " + dep.getGroup() + ":" + moduleName + " to " + dependencyName + " for license info");
-
             final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
             output.append(dep.getGroup() + ":" + moduleName + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
         }
 
         Files.writeString(outputFile.toPath(), output.toString(), StandardOpenOption.CREATE);
     }
 
-    @Input
-    @Optional
-    public abstract MapProperty<String, String> getMappings();
-
     /**
      * Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>
      * based on dependency coordinates.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java

@@ -8,10 +8,9 @@
  */
 package org.elasticsearch.gradle.internal.precommit;
 
+import org.elasticsearch.gradle.internal.AbstractDependenciesTask;
 import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer.LicenseInfo;
-import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
-import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.component.ComponentIdentifier;
 import org.gradle.api.file.Directory;
@@ -39,7 +38,6 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -94,7 +92,7 @@
  * comply with the license terms.
  */
 @CacheableTask
-public abstract class DependencyLicensesTask extends DefaultTask {
+public abstract class DependencyLicensesTask extends AbstractDependenciesTask {
 
     private final Pattern regex = Pattern.compile("-v?\\d+.*");
 
@@ -112,11 +110,6 @@ public abstract class DependencyLicensesTask extends DefaultTask {
      */
     private final DirectoryProperty licensesDir;
 
-    /**
-     * A map of patterns to prefix, used to find the LICENSE and NOTICE file.
-     */
-    private Map<String, String> mappings = new LinkedHashMap<>();
-
     /**
      * Names of dependencies whose shas should not exist.
      */
@@ -128,25 +121,6 @@ public abstract class DependencyLicensesTask extends DefaultTask {
     private LinkedHashSet<String> ignoreFiles = new LinkedHashSet<>();
     private ProjectLayout projectLayout;
 
-    /**
-     * Add a mapping from a regex pattern for the jar name, to a prefix to find
-     * the LICENSE and NOTICE file for that jar.
-     */
-    public void mapping(Map<String, String> props) {
-        String from = props.get("from");
-        if (from == null) {
-            throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
-        }
-        String to = props.get("to");
-        if (to == null) {
-            throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
-        }
-        if (props.size() > 2) {
-            throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
-        }
-        mappings.put(from, to);
-    }
-
     @Inject
     public DependencyLicensesTask(ObjectFactory objects, ProjectLayout projectLayout) {
         this.projectLayout = projectLayout;
@@ -267,7 +241,7 @@ private void checkDependencies(Map<String, Boolean> licenses, Map<String, Boolea
         for (File dependency : dependencies) {
             String jarName = dependency.getName();
             String depName = regex.matcher(jarName).replaceFirst("");
-            String dependencyName = getDependencyName(mappings, depName);
+            String dependencyName = getDependencyName(getMappings().get(), depName);
             logger.info("mapped dependency name {} to {} for license/notice check", depName, dependencyName);
             checkFile(dependencyName, jarName, licenses, "LICENSE");
             checkFile(dependencyName, jarName, notices, "NOTICE");
@@ -321,11 +295,6 @@ public LinkedHashSet<String> getIgnoreFiles() {
         return new LinkedHashSet<>(ignoreFiles);
     }
 
-    @Input
-    public LinkedHashMap<String, String> getMappings() {
-        return new LinkedHashMap<>(mappings);
-    }
-
     /**
      * Convencience method for configuring dependencies to be checked and ignoring transitive dependencies for now.
      * */

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java

@@ -11,6 +11,7 @@
 
 import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin;
 
+import org.gradle.api.artifacts.ArtifactView;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ResolvableDependencies;
 import org.gradle.api.artifacts.component.ComponentIdentifier;
@@ -31,6 +32,14 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
         Configuration configuration,
         Spec<ComponentIdentifier> componentFilter
     ) {
+        return createNonTransitiveArtifactsView(configuration, componentFilter).getFiles();
+    }
+
+    public static ArtifactView createNonTransitiveArtifactsView(Configuration configuration) {
+        return createNonTransitiveArtifactsView(configuration, identifier -> true);
+    }
+
+    public static ArtifactView createNonTransitiveArtifactsView(Configuration configuration, Spec<ComponentIdentifier> componentFilter) {
         ResolvableDependencies incoming = configuration.getIncoming();
         return incoming.artifactView(viewConfiguration -> {
             Provider<Set<ComponentIdentifier>> firstLevelDependencyComponents = incoming.getResolutionResult()
@@ -47,7 +56,7 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
             viewConfiguration.componentFilter(
                 new AndSpec<>(identifier -> firstLevelDependencyComponents.get().contains(identifier), componentFilter)
             );
-        }).getFiles();
+        });
     }
 
     /**

client/sniffer/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersTask
+import org.elasticsearch.gradle.internal.AbstractDependenciesTask
 
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
@@ -67,7 +68,7 @@ tasks.named('forbiddenApisTest').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
 
-tasks.named("dependencyLicenses").configure {
+tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: /http.*/, to: 'httpclient'
   mapping from: /commons-.*/, to: 'commons'
 }

distribution/build.gradle

@@ -83,6 +83,11 @@ tasks.register("generateDependenciesReport", ConcatFilesTask) {
     'https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/9/ubi-minimal-9-source.tar.gz'
   ]
   additionalLines << rhelUbiFields.join(',')
+  doLast {
+    if(target.text.readLines().size() < 100) {
+      throw new GradleException("Suspiciously low number of dependencies. Double check.")
+    }
+  }
 }
 
 /*****************************************************************************

modules/ingest-geoip/build.gradle

@@ -8,6 +8,7 @@
  */
 
 import org.elasticsearch.gradle.OS
+import org.elasticsearch.gradle.internal.AbstractDependenciesTask
 
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
@@ -81,10 +82,13 @@ tasks.named("forbiddenPatterns").configure {
   exclude '**/*.mmdb'
 }
 
-tasks.named("dependencyLicenses").configure {
+tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: /geoip.*/, to: 'maxmind-geolite2-eula'
   mapping from: /maxmind-db.*/, to: 'maxmind-db-reader'
   mapping from: /jackson.*/, to: 'jackson'
+}
+
+tasks.named("dependencyLicenses").configure {
   ignoreFile 'elastic-geoip-database-service-agreement-LICENSE.txt'
 }
 

modules/repository-s3/build.gradle

@@ -8,6 +8,7 @@
  */
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
+import org.elasticsearch.gradle.internal.AbstractDependenciesTask
 
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
@@ -89,7 +90,7 @@ restResources {
   }
 }
 
-tasks.named("dependencyLicenses").configure {
+tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: 'annotations',              to: 'aws-sdk-2'
   mapping from: 'apache-client',            to: 'aws-sdk-2'
   mapping from: 'arns',                     to: 'aws-sdk-2'