Fix dependency reporting

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/BuildPluginFuncTest.groovy

@@ -151,7 +151,11 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
             tasks.named('checkstyleMain').configure { enabled = false }
             tasks.named('loggerUsageCheck').configure { enabled = false }
             // tested elsewhere
-            tasks.named('thirdPartyAudit').configure { enabled = false }
+            tasks.named('thirdPartyAudit').configure {
+                getRuntimeJavaVersion().set(JavaVersion.VERSION_21)
+                getTargetCompatibility().set(JavaVersion.VERSION_21)
+                enabled = false
+            }
             """
         when:
         def result = gradleRunner("check").build()
@@ -167,6 +171,25 @@ class BuildPluginFuncTest extends AbstractGradleFuncTest {
         result.task(":loggerUsageCheck").outcome == TaskOutcome.SKIPPED
     }
 
+    def "can generate dependency infos file"() {
+        given:
+        repository.generateJar("junit", "junit", "4.12", 'org.acme.JunitMock')
+        repository.configureBuild(buildFile)
+        file("licenses/junit-4.12.jar.sha1").text = "2973d150c0dc1fefe998f834810d68f278ea58ec"
+        file("licenses/junit-LICENSE.txt").text = EXAMPLE_LICENSE
+        file("licenses/junit-NOTICE.txt").text = "mock notice"
+        buildFile << """
+        dependencies {
+            api "junit:junit:4.12"
+        }
+        """
+        when:
+        def result = gradleRunner("dependenciesInfo").build()
+        then:
+        result.task(":dependenciesInfo").outcome == TaskOutcome.SUCCESS
+        file("build/reports/dependencies/dependencies.csv").text == "junit:junit,4.12,https://repo1.maven.org/maven2/junit/junit/4.12,BSD-3-Clause,\n"
+    }
+
     def assertValidJar(File jar) {
         try (ZipFile zipFile = new ZipFile(jar)) {
             ZipEntry licenseEntry = zipFile.getEntry("META-INF/LICENSE.txt")

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/AbstractDependenciesTask.java

@@ -0,0 +1,44 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.gradle.internal;
+
+import org.gradle.api.DefaultTask;
+import org.gradle.api.InvalidUserDataException;
+import org.gradle.api.provider.MapProperty;
+import org.gradle.api.tasks.Input;
+import org.gradle.api.tasks.Optional;
+
+import java.util.Map;
+
+public abstract class AbstractDependenciesTask extends DefaultTask {
+
+    @Input
+    @Optional
+    public abstract MapProperty<String, String> getMappings();
+
+    /**
+     * Add a mapping from a regex pattern for the jar name, to a prefix to find
+     * the LICENSE and NOTICE file for that jar.
+     */
+    public void mapping(Map<String, String> props) {
+        String from = props.get("from");
+        if (from == null) {
+            throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
+        }
+        String to = props.get("to");
+        if (to == null) {
+            throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
+        }
+        if (props.size() > 2) {
+            throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
+        }
+        getMappings().put(from, to);
+    }
+}

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoPlugin.java

@@ -17,6 +17,8 @@
 import org.gradle.api.attributes.Usage;
 import org.gradle.api.plugins.JavaPlugin;
 
+import static org.elasticsearch.gradle.internal.util.DependenciesUtils.createNonTransitiveArtifactsView;
+
 public class DependenciesInfoPlugin implements Plugin<Project> {
 
     public static String USAGE_ATTRIBUTE = "DependenciesInfo";
@@ -25,16 +27,15 @@ public class DependenciesInfoPlugin implements Plugin<Project> {
     public void apply(final Project project) {
         project.getPlugins().apply(CompileOnlyResolvePlugin.class);
         var depsInfo = project.getTasks().register("dependenciesInfo", DependenciesInfoTask.class);
-
         depsInfo.configure(t -> {
             var runtimeConfiguration = project.getConfigurations().getByName(JavaPlugin.RUNTIME_CLASSPATH_CONFIGURATION_NAME);
-            t.getRuntimeArtifacts().set(project.getProviders().provider(() -> runtimeConfiguration.getIncoming().getArtifacts()));
+            t.getRuntimeArtifacts()
+                .set(project.getProviders().provider(() -> createNonTransitiveArtifactsView(runtimeConfiguration).getArtifacts()));
             t.getClasspath().from(runtimeConfiguration);
             var compileOnlyConfiguration = project.getConfigurations()
                 .getByName(CompileOnlyResolvePlugin.RESOLVEABLE_COMPILE_ONLY_CONFIGURATION_NAME);
             t.getCompileOnlyArtifacts().set(project.getProviders().provider(() -> compileOnlyConfiguration.getIncoming().getArtifacts()));
             t.getClasspath().from(compileOnlyConfiguration);
-
         });
         Configuration dependenciesInfoFilesConfiguration = project.getConfigurations().create("dependenciesInfoFiles");
         dependenciesInfoFilesConfiguration.setCanBeResolved(false);

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/DependenciesInfoTask.java

@@ -17,19 +17,21 @@
 import org.gradle.api.file.ConfigurableFileCollection;
 import org.gradle.api.file.DirectoryProperty;
 import org.gradle.api.file.ProjectLayout;
-import org.gradle.api.internal.ConventionTask;
 import org.gradle.api.model.ObjectFactory;
-import org.gradle.api.provider.MapProperty;
 import org.gradle.api.provider.Property;
 import org.gradle.api.provider.Provider;
 import org.gradle.api.provider.ProviderFactory;
+import org.gradle.api.tasks.CacheableTask;
+import org.gradle.api.tasks.Classpath;
 import org.gradle.api.tasks.Input;
 import org.gradle.api.tasks.InputDirectory;
-import org.gradle.api.tasks.InputFiles;
 import org.gradle.api.tasks.Internal;
 import org.gradle.api.tasks.Optional;
 import org.gradle.api.tasks.OutputFile;
+import org.gradle.api.tasks.PathSensitive;
+import org.gradle.api.tasks.PathSensitivity;
 import org.gradle.api.tasks.TaskAction;
+import org.gradle.internal.component.external.model.ModuleComponentArtifactIdentifier;
 
 import java.io.File;
 import java.io.IOException;
@@ -55,7 +57,8 @@
  *     <li>license: <a href="https://spdx.org/licenses/">SPDX license</a> identifier, custom license or UNKNOWN.</li>
  * </ul>
  */
-public abstract class DependenciesInfoTask extends ConventionTask {
+@CacheableTask
+public abstract class DependenciesInfoTask extends AbstractDependenciesTask {
 
     @Inject
     public abstract ProviderFactory getProviderFactory();
@@ -86,16 +89,17 @@ public Provider<Set<ModuleComponentIdentifier>> getCompileOnlyModules() {
      * artifact transforms that might be applied and fail due to missing task dependency to jar
      * generating tasks.
      * */
-    @InputFiles
+    @Classpath
     abstract ConfigurableFileCollection getClasspath();
 
     private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers(ArtifactCollection artifacts) {
         return getProviderFactory().provider(
             () -> artifacts.getArtifacts()
                 .stream()
                 .map(r -> r.getId())
-                .filter(id -> id instanceof ModuleComponentIdentifier)
-                .map(id -> (ModuleComponentIdentifier) id)
+                .filter(mcaId -> mcaId instanceof ModuleComponentArtifactIdentifier)
+                .map(mcaId -> (ModuleComponentArtifactIdentifier) mcaId)
+                .map(it -> it.getComponentIdentifier())
                 .collect(Collectors.toSet())
         );
     }
@@ -111,6 +115,7 @@ private Provider<Set<ModuleComponentIdentifier>> mapToModuleComponentIdentifiers
      * Directory to read license files
      */
     @Optional
+    @PathSensitive(PathSensitivity.RELATIVE)
     @InputDirectory
     public File getLicensesDir() {
         File asFile = licensesDir.get().getAsFile();
@@ -143,7 +148,6 @@ public DependenciesInfoTask(ProjectLayout projectLayout, ObjectFactory objectFac
 
     @TaskAction
     public void generateDependenciesInfo() throws IOException {
-
         final Set<String> compileOnlyIds = getCompileOnlyModules().map(
             set -> set.stream()
                 .map(id -> id.getModuleIdentifier().getGroup() + ":" + id.getModuleIdentifier().getName() + ":" + id.getVersion())
@@ -166,18 +170,13 @@ public void generateDependenciesInfo() throws IOException {
             final String url = createURL(dep.getGroup(), moduleName, dep.getVersion());
             final String dependencyName = DependencyLicensesTask.getDependencyName(mappings, moduleName);
             getLogger().info("mapped dependency " + dep.getGroup() + ":" + moduleName + " to " + dependencyName + " for license info");
-
             final String licenseType = getLicenseType(dep.getGroup(), dependencyName);
             output.append(dep.getGroup() + ":" + moduleName + "," + dep.getVersion() + "," + url + "," + licenseType + "\n");
         }
 
         Files.writeString(outputFile.toPath(), output.toString(), StandardOpenOption.CREATE);
     }
 
-    @Input
-    @Optional
-    public abstract MapProperty<String, String> getMappings();
-
     /**
      * Create an URL on <a href="https://repo1.maven.org/maven2/">Maven Central</a>
      * based on dependency coordinates.

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/precommit/DependencyLicensesTask.java

@@ -8,10 +8,9 @@
  */
 package org.elasticsearch.gradle.internal.precommit;
 
+import org.elasticsearch.gradle.internal.AbstractDependenciesTask;
 import org.elasticsearch.gradle.internal.precommit.LicenseAnalyzer.LicenseInfo;
-import org.gradle.api.DefaultTask;
 import org.gradle.api.GradleException;
-import org.gradle.api.InvalidUserDataException;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.component.ComponentIdentifier;
 import org.gradle.api.file.Directory;
@@ -39,7 +38,6 @@
 import java.util.Arrays;
 import java.util.HashMap;
 import java.util.HashSet;
-import java.util.LinkedHashMap;
 import java.util.LinkedHashSet;
 import java.util.List;
 import java.util.Map;
@@ -94,7 +92,7 @@
  * comply with the license terms.
  */
 @CacheableTask
-public abstract class DependencyLicensesTask extends DefaultTask {
+public abstract class DependencyLicensesTask extends AbstractDependenciesTask {
 
     private final Pattern regex = Pattern.compile("-v?\\d+.*");
 
@@ -112,11 +110,6 @@ public abstract class DependencyLicensesTask extends DefaultTask {
      */
     private final DirectoryProperty licensesDir;
 
-    /**
-     * A map of patterns to prefix, used to find the LICENSE and NOTICE file.
-     */
-    private Map<String, String> mappings = new LinkedHashMap<>();
-
     /**
      * Names of dependencies whose shas should not exist.
      */
@@ -128,25 +121,6 @@ public abstract class DependencyLicensesTask extends DefaultTask {
     private LinkedHashSet<String> ignoreFiles = new LinkedHashSet<>();
     private ProjectLayout projectLayout;
 
-    /**
-     * Add a mapping from a regex pattern for the jar name, to a prefix to find
-     * the LICENSE and NOTICE file for that jar.
-     */
-    public void mapping(Map<String, String> props) {
-        String from = props.get("from");
-        if (from == null) {
-            throw new InvalidUserDataException("Missing \"from\" setting for license name mapping");
-        }
-        String to = props.get("to");
-        if (to == null) {
-            throw new InvalidUserDataException("Missing \"to\" setting for license name mapping");
-        }
-        if (props.size() > 2) {
-            throw new InvalidUserDataException("Unknown properties for mapping on dependencyLicenses: " + props.keySet());
-        }
-        mappings.put(from, to);
-    }
-
     @Inject
     public DependencyLicensesTask(ObjectFactory objects, ProjectLayout projectLayout) {
         this.projectLayout = projectLayout;
@@ -267,7 +241,7 @@ private void checkDependencies(Map<String, Boolean> licenses, Map<String, Boolea
         for (File dependency : dependencies) {
             String jarName = dependency.getName();
             String depName = regex.matcher(jarName).replaceFirst("");
-            String dependencyName = getDependencyName(mappings, depName);
+            String dependencyName = getDependencyName(getMappings().get(), depName);
             logger.info("mapped dependency name {} to {} for license/notice check", depName, dependencyName);
             checkFile(dependencyName, jarName, licenses, "LICENSE");
             checkFile(dependencyName, jarName, notices, "NOTICE");
@@ -321,11 +295,6 @@ public LinkedHashSet<String> getIgnoreFiles() {
         return new LinkedHashSet<>(ignoreFiles);
     }
 
-    @Input
-    public LinkedHashMap<String, String> getMappings() {
-        return new LinkedHashMap<>(mappings);
-    }
-
     /**
      * Convencience method for configuring dependencies to be checked and ignoring transitive dependencies for now.
      * */

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/util/DependenciesUtils.java

@@ -11,6 +11,7 @@
 
 import com.github.jengelman.gradle.plugins.shadow.ShadowBasePlugin;
 
+import org.gradle.api.artifacts.ArtifactView;
 import org.gradle.api.artifacts.Configuration;
 import org.gradle.api.artifacts.ResolvableDependencies;
 import org.gradle.api.artifacts.component.ComponentIdentifier;
@@ -31,6 +32,14 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
         Configuration configuration,
         Spec<ComponentIdentifier> componentFilter
     ) {
+        return createNonTransitiveArtifactsView(configuration, componentFilter).getFiles();
+    }
+
+    public static ArtifactView createNonTransitiveArtifactsView(Configuration configuration) {
+        return createNonTransitiveArtifactsView(configuration, identifier -> true);
+    }
+
+    public static ArtifactView createNonTransitiveArtifactsView(Configuration configuration, Spec<ComponentIdentifier> componentFilter) {
         ResolvableDependencies incoming = configuration.getIncoming();
         return incoming.artifactView(viewConfiguration -> {
             Provider<Set<ComponentIdentifier>> firstLevelDependencyComponents = incoming.getResolutionResult()
@@ -47,7 +56,7 @@ public static FileCollection createFileCollectionFromNonTransitiveArtifactsView(
             viewConfiguration.componentFilter(
                 new AndSpec<>(identifier -> firstLevelDependencyComponents.get().contains(identifier), componentFilter)
             );
-        }).getFiles();
+        });
     }
 
     /**

client/sniffer/build.gradle

@@ -17,6 +17,7 @@
  * under the License.
  */
 import org.elasticsearch.gradle.internal.conventions.precommit.LicenseHeadersTask
+import org.elasticsearch.gradle.internal.AbstractDependenciesTask
 
 apply plugin: 'elasticsearch.build'
 apply plugin: 'elasticsearch.publish'
@@ -67,7 +68,7 @@ tasks.named('forbiddenApisTest').configure {
   replaceSignatureFiles 'jdk-signatures'
 }
 
-tasks.named("dependencyLicenses").configure {
+tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: /http.*/, to: 'httpclient'
   mapping from: /commons-.*/, to: 'commons'
 }

distribution/build.gradle

@@ -83,6 +83,11 @@ tasks.register("generateDependenciesReport", ConcatFilesTask) {
     'https://oss-dependencies.elastic.co/red-hat-universal-base-image-minimal/9/ubi-minimal-9-source.tar.gz'
   ]
   additionalLines << rhelUbiFields.join(',')
+  doLast {
+    if(target.text.readLines().size() < 100) {
+      throw new GradleException("Suspiciously low number of dependencies. Double check.")
+    }
+  }
 }
 
 /*****************************************************************************

modules/ingest-geoip/build.gradle

@@ -8,6 +8,7 @@
  */
 
 import org.elasticsearch.gradle.OS
+import org.elasticsearch.gradle.internal.AbstractDependenciesTask
 
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.yaml-rest-compat-test'
@@ -81,10 +82,13 @@ tasks.named("forbiddenPatterns").configure {
   exclude '**/*.mmdb'
 }
 
-tasks.named("dependencyLicenses").configure {
+tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: /geoip.*/, to: 'maxmind-geolite2-eula'
   mapping from: /maxmind-db.*/, to: 'maxmind-db-reader'
   mapping from: /jackson.*/, to: 'jackson'
+}
+
+tasks.named("dependencyLicenses").configure {
   ignoreFile 'elastic-geoip-database-service-agreement-LICENSE.txt'
 }
 

modules/repository-s3/build.gradle

@@ -8,6 +8,7 @@
  */
 import org.apache.tools.ant.filters.ReplaceTokens
 import org.elasticsearch.gradle.internal.test.InternalClusterTestPlugin
+import org.elasticsearch.gradle.internal.AbstractDependenciesTask
 
 apply plugin: 'elasticsearch.internal-yaml-rest-test'
 apply plugin: 'elasticsearch.internal-cluster-test'
@@ -89,7 +90,7 @@ restResources {
   }
 }
 
-tasks.named("dependencyLicenses").configure {
+tasks.withType(AbstractDependenciesTask).configureEach {
   mapping from: 'annotations',              to: 'aws-sdk-2'
   mapping from: 'apache-client',            to: 'aws-sdk-2'
   mapping from: 'arns',                     to: 'aws-sdk-2'