[9.0] [Gradle] Configure transitive dependencies via ComponentMetadataRules (#134169)

BUILDING.md

@@ -97,6 +97,70 @@ will have the `origin` attribute been set to `Generated by Gradle`.
 >Please replace the content of the `origin` attribute by `official site` in that case.
 >
 
+##### Handling transitive dependencies
+
+Dependency management is a critical aspect of maintaining a secure and reliable build system, requiring explicit control over what we rely on. The Elasticsearch build mainly uses component metadata rules declared in the `ComponentMetadataRulesPlugin`
+plugin to manage transitive dependencies and avoid version conflicts.
+This approach ensures we have explicit control over all dependencies used in the build.
+
+###### General Guidelines
+
+1. **Avoid unused transitive dependencies** - Dependencies that are not actually used by our code should be excluded to reduce the attack surface and avoid potential conflicts.
+
+2. **Prefer versions declared in `build-tools-internal/version.properties`** - All dependency versions should be centrally managed in this file to ensure consistency across the entire build.
+
+3. **Libraries required to compile our code should be direct dependencies** - If we directly use a library in our source code, it should be declared as a direct dependency rather than relying on it being transitively available.
+
+###### Component Metadata Rules
+
+We use two main types of component metadata rules at this point to manage transitive dependencies:
+
+- **`ExcludeAllTransitivesRule`** - Excludes all transitive dependencies for libraries where we want complete control over dependencies or the transitive dependencies are unused.
+
+- **`ExcludeOtherGroupsTransitiveRule`** - Excludes transitive dependencies that don't belong to the same group as the direct dependency, while keeping same-group dependencies.
+-
+- **`ExcludeByGroup`** - Excludes transitive dependencies that match a specific groupId while keeping all other transitive dependencies with different groupIds.
+
+Examples from the `ComponentMetadataRulesPlugin`:
+
+```gradle
+// Exclude all transitives - used when transitive deps are unused or problematic
+components.withModule("com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", ExcludeAllTransitivesRule.class);
+
+// Exclude other groups - used when we want same-group deps but not external ones
+components.withModule("com.azure:azure-core", ExcludeOtherGroupsTransitiveRule.class);
+
+// Exclude only specific groups - used when we want exclude specific group of transitive deps.
+components.withModule("org.apache.logging.log4j:log4j-api", ExcludeByGroup.class, rule -> {
+    rule.params(List.of("biz.aQute.bnd", "org.osgi"));
+});
+```
+
+###### Common Scenarios
+
+**Version Conflicts**: When a transitive dependency brings in a different version than what we use:
+```gradle
+// brings in jackson-databind and jackson-annotations not used
+components.withModule("com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", ExcludeAllTransitivesRule.class);
+```
+
+**Unused Dependencies**: When transitive dependencies are not actually used:
+```gradle
+// brings in azure-core-http-netty. not used
+components.withModule("com.azure:azure-core-http-netty", ExcludeAllTransitivesRule.class);
+```
+
+**Mismatching Version Dependencies**: When other versions are required:
+```gradle
+// brings in org.slf4j:slf4j-api:1.7.25. We use 2.0.6
+components.withModule("org.apache.directory.api:api-asn1-ber", ExcludeOtherGroupsTransitiveRule.class);
+```
+
+When adding or updating dependencies, ensure that any required transitive dependencies are either:
+1. Already available as direct dependencies with compatible versions
+2. Added as direct dependencies if they're actually used by our code
+3. Properly excluded if they're not needed
+
 #### Custom plugin and task implementations
 
 Build logic that is used across multiple subprojects should be considered to be moved into a Gradle plugin with according Gradle task implementation.

build-conventions/build.gradle

@@ -72,6 +72,9 @@ repositories {
 }
 
 dependencies {
+  constraints {
+    api("org.slf4j:slf4j-api:2.0.6")
+  }
   api buildLibs.maven.model
   api buildLibs.shadow.plugin
   api buildLibs.apache.rat

build-tools-internal/build.gradle

@@ -40,6 +40,10 @@ gradlePlugin {
       id = 'elasticsearch.build-complete'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchBuildCompletePlugin'
     }
+    componentMetadataRules {
+      id = 'elasticsearch.component-metadata-rules'
+      implementationClass = 'org.elasticsearch.gradle.internal.dependencies.rules.ComponentMetadataRulesPlugin'
+    }
     distro {
       id = 'elasticsearch.distro'
       implementationClass = 'org.elasticsearch.gradle.internal.distribution.ElasticsearchDistributionPlugin'
@@ -281,6 +285,9 @@ dependencies {
     testImplementation buildLibs.asm
     integTestImplementation buildLibs.asm
     api(buildLibs.snakeyaml)
+    api("org.slf4j:slf4j-api:2.0.6") {
+      because("Align with what we use in production")
+    }
   }
   // Forcefully downgrade the jackson platform as used in production
   api enforcedPlatform(buildLibs.jackson.platform)

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -16,18 +16,14 @@
 import org.elasticsearch.gradle.internal.test.MutedTestPlugin;
 import org.elasticsearch.gradle.internal.test.TestUtil;
 import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
-import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.ResolutionStrategy;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.Provider;
-import org.gradle.api.tasks.SourceSet;
-import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.compile.AbstractCompile;
 import org.gradle.api.tasks.compile.CompileOptions;
 import org.gradle.api.tasks.compile.GroovyCompile;
@@ -67,8 +63,6 @@ public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
         project.getPluginManager().apply(PrecommitTaskPlugin.class);
         project.getPluginManager().apply(MutedTestPlugin.class);
-
-        configureConfigurations(project);
         configureCompile(project);
         configureInputNormalization(project);
         configureNativeLibraryPath(project);
@@ -77,54 +71,6 @@ public void apply(Project project) {
         project.getExtensions().getExtraProperties().set("versions", VersionProperties.getVersions());
     }
 
-    /**
-     * Makes dependencies non-transitive.
-     * <p>
-     * Gradle allows setting all dependencies as non-transitive very easily.
-     * Sadly this mechanism does not translate into maven pom generation. In order
-     * to effectively make the pom act as if it has no transitive dependencies,
-     * we must exclude each transitive dependency of each direct dependency.
-     * <p>
-     * Determining the transitive deps of a dependency which has been resolved as
-     * non-transitive is difficult because the process of resolving removes the
-     * transitive deps. To sidestep this issue, we create a configuration per
-     * direct dependency version. This specially named and unique configuration
-     * will contain all of the transitive dependencies of this particular
-     * dependency. We can then use this configuration during pom generation
-     * to iterate the transitive dependencies and add excludes.
-     */
-    public static void configureConfigurations(Project project) {
-        // we are not shipping these jars, we act like dumb consumers of these things
-        if (project.getPath().startsWith(":test:fixtures") || project.getPath().equals(":build-tools")) {
-            return;
-        }
-        // fail on any conflicting dependency versions
-        project.getConfigurations().all(configuration -> {
-            if (configuration.getName().endsWith("Fixture")) {
-                // just a self contained test-fixture configuration, likely transitive and hellacious
-                return;
-            }
-            configuration.resolutionStrategy(ResolutionStrategy::failOnVersionConflict);
-        });
-
-        // disable transitive dependency management
-        SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-        sourceSets.all(sourceSet -> disableTransitiveDependenciesForSourceSet(project, sourceSet));
-    }
-
-    private static void disableTransitiveDependenciesForSourceSet(Project project, SourceSet sourceSet) {
-        List<String> sourceSetConfigurationNames = List.of(
-            sourceSet.getApiConfigurationName(),
-            sourceSet.getImplementationConfigurationName(),
-            sourceSet.getCompileOnlyConfigurationName(),
-            sourceSet.getRuntimeOnlyConfigurationName()
-        );
-
-        project.getConfigurations()
-            .matching(c -> sourceSetConfigurationNames.contains(c.getName()))
-            .configureEach(GradleUtils::disableTransitiveDependencies);
-    }
-
     /**
      * Adds compiler settings to the project
      */

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaModulePathPlugin.java

@@ -21,6 +21,7 @@
 import org.gradle.api.artifacts.result.ResolvedComponentResult;
 import org.gradle.api.artifacts.result.ResolvedDependencyResult;
 import org.gradle.api.attributes.LibraryElements;
+import org.gradle.api.attributes.Usage;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.logging.Logger;
 import org.gradle.api.plugins.JavaPlugin;
@@ -75,12 +76,13 @@ void configureCompileModulePath(Project project) {
             it.extendsFrom(compileClasspath);
             it.setCanBeResolved(true);
             it.setCanBeConsumed(false); // we don't want this configuration used by dependent projects
-            it.attributes(
-                attrs -> attrs.attribute(
+            it.attributes(attrs -> {
+                attrs.attribute(
                     LibraryElements.LIBRARY_ELEMENTS_ATTRIBUTE,
                     project.getObjects().named(LibraryElements.class, LibraryElements.CLASSES)
-                )
-            );
+                );
+                attrs.attribute(Usage.USAGE_ATTRIBUTE, project.getObjects().named(Usage.class, Usage.JAVA_API));
+            });
         }).getIncoming().artifactView(it -> {
             it.componentFilter(cf -> {
                 var visited = new HashSet<ComponentIdentifier>();