Merge remote-tracking branch 'origin/main' into lucene_snapshot_10_3

Author: romseygeek

.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap

@@ -309,3 +309,68 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha
   },
 ]
 `;
+
+exports[`generatePipelines should generate correct pipelines with a different branch that is not skipped 1`] = `
+[
+  {
+    "name": "bwc-snapshots",
+    "pipeline": {
+      "steps": [
+        {
+          "group": "bwc-snapshots",
+          "steps": [
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2404",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
+              "env": {
+                "BWC_VERSION": "{{matrix.BWC_VERSION}}",
+              },
+              "label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
+              "matrix": {
+                "setup": {
+                  "BWC_VERSION": [
+                    "7.17.14",
+                    "8.10.3",
+                    "8.11.0",
+                  ],
+                },
+              },
+              "timeout_in_minutes": 300,
+            },
+          ],
+        },
+      ],
+    },
+  },
+  {
+    "name": "skip-branch",
+    "pipeline": {
+      "steps": [
+        {
+          "command": "echo 1",
+          "label": "skip-me",
+        },
+      ],
+    },
+  },
+  {
+    "name": "using-defaults",
+    "pipeline": {
+      "env": {
+        "CUSTOM_ENV_VAR": "value",
+      },
+      "steps": [
+        {
+          "command": "echo 'hello world'",
+          "label": "test-step",
+        },
+      ],
+    },
+  },
+]
+`;

.buildkite/scripts/pull-request/mocks/pipelines/skip-branch.yml

@@ -0,0 +1,5 @@
+config:
+  skip-target-branches: "test-branch"
+steps:
+  - label: skip-me
+    command: echo 1

.buildkite/scripts/pull-request/pipeline.test.ts

@@ -8,6 +8,7 @@ describe("generatePipelines", () => {
     setBwcVersionsPath(`${import.meta.dir}/mocks/bwcVersions`);
     setSnapshotBwcVersionsPath(`${import.meta.dir}/mocks/snapshotBwcVersions`);
 
+    process.env["GITHUB_PR_TARGET_BRANCH"] = "test-branch";
     process.env["GITHUB_PR_LABELS"] = "test-label-1,test-label-2";
     process.env["GITHUB_PR_TRIGGER_COMMENT"] = "";
   });
@@ -36,6 +37,12 @@ describe("generatePipelines", () => {
     testWithTriggerCheck(`${import.meta.dir}/mocks/pipelines`, ["build.gradle"]);
   });
 
+  test("should generate correct pipelines with a different branch that is not skipped", () => {
+    process.env["GITHUB_PR_TARGET_BRANCH"] = "main";
+
+    testWithTriggerCheck(`${import.meta.dir}/mocks/pipelines`, ["build.gradle"]);
+  });
+
   test("should generate correct pipeline when using a trigger comment for it", () => {
     process.env["GITHUB_PR_TRIGGER_COMMENT"] = "run elasticsearch-ci/using-defaults";
 

.buildkite/scripts/pull-request/pipeline.ts

@@ -50,6 +50,14 @@ const changedFilesIncludedCheck = (pipeline: EsPipeline, changedFiles: string[])
   return true;
 };
 
+const checkTargetBranch = (pipeline: EsPipeline, targetBranch: string | undefined) => {
+  if (!targetBranch || !pipeline.config?.["skip-target-branches"]) {
+    return true;
+  }
+
+  return !getArray(pipeline.config["skip-target-branches"]).some((branch) => branch === targetBranch);
+};
+
 const triggerCommentCheck = (pipeline: EsPipeline): boolean => {
   if (process.env["GITHUB_PR_TRIGGER_COMMENT"] && pipeline.config?.["trigger-phrase"]) {
     return !!process.env["GITHUB_PR_TRIGGER_COMMENT"].match(pipeline.config["trigger-phrase"]);
@@ -138,6 +146,7 @@ export const generatePipelines = (
   }
 
   let filters: ((pipeline: EsPipeline) => boolean)[] = [
+    (pipeline) => checkTargetBranch(pipeline, process.env["GITHUB_PR_TARGET_BRANCH"]),
     (pipeline) => labelCheckAllow(pipeline, labels),
     (pipeline) => labelCheckSkip(pipeline, labels),
     (pipeline) => changedFilesExcludedCheck(pipeline, changedFiles),

.buildkite/scripts/pull-request/types.ts

@@ -5,6 +5,7 @@ export type EsPipelineConfig = {
     "included-regions"?: string | string[];
     "excluded-regions"?: string | string[];
     "trigger-phrase"?: string;
+    "skip-target-branches"?: string | string[];
   };
 };
 

.buildkite/scripts/validate-transport-version-backport.sh

@@ -1,6 +1,11 @@
 #!/bin/bash
 set -euo pipefail
 
+if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" == "main" ]]; then
+  # Don't run on PRs targeting main
+  exit 0
+fi
+
 echo "--- Looking for transport version changes"
 
 # Get any changes in this pull request to transport definitions

BUILDING.md

@@ -97,6 +97,70 @@ will have the `origin` attribute been set to `Generated by Gradle`.
 > If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
 > Please replace the content of the `origin` attribute by `official site` in that case.
 
+##### Handling transitive dependencies
+
+Dependency management is a critical aspect of maintaining a secure and reliable build system, requiring explicit control over what we rely on. The Elasticsearch build mainly uses component metadata rules declared in the `ComponentMetadataRulesPlugin`
+plugin to manage transitive dependencies and avoid version conflicts.
+This approach ensures we have explicit control over all dependencies used in the build.
+
+###### General Guidelines
+
+1. **Avoid unused transitive dependencies** - Dependencies that are not actually used by our code should be excluded to reduce the attack surface and avoid potential conflicts.
+
+2. **Prefer versions declared in `build-tools-internal/version.properties`** - All dependency versions should be centrally managed in this file to ensure consistency across the entire build.
+
+3. **Libraries required to compile our code should be direct dependencies** - If we directly use a library in our source code, it should be declared as a direct dependency rather than relying on it being transitively available.
+
+###### Component Metadata Rules
+
+We use two main types of component metadata rules at this point to manage transitive dependencies:
+
+- **`ExcludeAllTransitivesRule`** - Excludes all transitive dependencies for libraries where we want complete control over dependencies or the transitive dependencies are unused.
+
+- **`ExcludeOtherGroupsTransitiveRule`** - Excludes transitive dependencies that don't belong to the same group as the direct dependency, while keeping same-group dependencies.
+-
+- **`ExcludeByGroup`** - Excludes transitive dependencies that match a specific groupId while keeping all other transitive dependencies with different groupIds.
+
+Examples from the `ComponentMetadataRulesPlugin`:
+
+```gradle
+// Exclude all transitives - used when transitive deps are unused or problematic
+components.withModule("com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", ExcludeAllTransitivesRule.class);
+
+// Exclude other groups - used when we want same-group deps but not external ones
+components.withModule("com.azure:azure-core", ExcludeOtherGroupsTransitiveRule.class);
+
+// Exclude only specific groups - used when we want exclude specific group of transitive deps.
+components.withModule("org.apache.logging.log4j:log4j-api", ExcludeByGroup.class, rule -> {
+    rule.params(List.of("biz.aQute.bnd", "org.osgi"));
+});
+```
+
+###### Common Scenarios
+
+**Version Conflicts**: When a transitive dependency brings in a different version than what we use:
+```gradle
+// brings in jackson-databind and jackson-annotations not used
+components.withModule("com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", ExcludeAllTransitivesRule.class);
+```
+
+**Unused Dependencies**: When transitive dependencies are not actually used:
+```gradle
+// brings in azure-core-http-netty. not used
+components.withModule("com.azure:azure-core-http-netty", ExcludeAllTransitivesRule.class);
+```
+
+**Mismatching Version Dependencies**: When other versions are required:
+```gradle
+// brings in org.slf4j:slf4j-api:1.7.25. We use 2.0.6
+components.withModule("org.apache.directory.api:api-asn1-ber", ExcludeOtherGroupsTransitiveRule.class);
+```
+
+When adding or updating dependencies, ensure that any required transitive dependencies are either:
+1. Already available as direct dependencies with compatible versions
+2. Added as direct dependencies if they're actually used by our code
+3. Properly excluded if they're not needed
+
 #### Custom plugin and task implementations
 
 Build logic that is used across multiple subprojects should be considered to be moved into a Gradle plugin with according Gradle task implementation.

build-conventions/build.gradle

@@ -72,6 +72,9 @@ repositories {
 }
 
 dependencies {
+  constraints {
+    api("org.slf4j:slf4j-api:2.0.6")
+  }
   api buildLibs.maven.model
   api buildLibs.shadow.plugin
   api buildLibs.apache.rat

build-tools-internal/build.gradle

@@ -40,6 +40,10 @@ gradlePlugin {
       id = 'elasticsearch.build-complete'
       implementationClass = 'org.elasticsearch.gradle.internal.ElasticsearchBuildCompletePlugin'
     }
+    componentMetadataRules {
+      id = 'elasticsearch.component-metadata-rules'
+      implementationClass = 'org.elasticsearch.gradle.internal.dependencies.rules.ComponentMetadataRulesPlugin'
+    }
     distro {
       id = 'elasticsearch.distro'
       implementationClass = 'org.elasticsearch.gradle.internal.distribution.ElasticsearchDistributionPlugin'
@@ -281,6 +285,9 @@ dependencies {
     testImplementation buildLibs.asm
     integTestImplementation buildLibs.asm
     api(buildLibs.snakeyaml)
+    api("org.slf4j:slf4j-api:2.0.6") {
+      because("Align with what we use in production")
+    }
   }
   // Forcefully downgrade the jackson platform as used in production
   api enforcedPlatform(buildLibs.jackson.platform)

build-tools-internal/src/main/java/org/elasticsearch/gradle/internal/ElasticsearchJavaBasePlugin.java

@@ -16,18 +16,14 @@
 import org.elasticsearch.gradle.internal.test.MutedTestPlugin;
 import org.elasticsearch.gradle.internal.test.TestUtil;
 import org.elasticsearch.gradle.test.SystemPropertyCommandLineArgumentProvider;
-import org.elasticsearch.gradle.util.GradleUtils;
 import org.gradle.api.JavaVersion;
 import org.gradle.api.Plugin;
 import org.gradle.api.Project;
 import org.gradle.api.artifacts.Configuration;
-import org.gradle.api.artifacts.ResolutionStrategy;
 import org.gradle.api.file.FileCollection;
 import org.gradle.api.plugins.JavaBasePlugin;
 import org.gradle.api.plugins.JavaPluginExtension;
 import org.gradle.api.provider.Provider;
-import org.gradle.api.tasks.SourceSet;
-import org.gradle.api.tasks.SourceSetContainer;
 import org.gradle.api.tasks.compile.AbstractCompile;
 import org.gradle.api.tasks.compile.CompileOptions;
 import org.gradle.api.tasks.compile.GroovyCompile;
@@ -67,8 +63,6 @@ public void apply(Project project) {
         project.getPluginManager().apply(ElasticsearchTestBasePlugin.class);
         project.getPluginManager().apply(PrecommitTaskPlugin.class);
         project.getPluginManager().apply(MutedTestPlugin.class);
-
-        configureConfigurations(project);
         configureCompile(project);
         configureInputNormalization(project);
         configureNativeLibraryPath(project);
@@ -77,54 +71,6 @@ public void apply(Project project) {
         project.getExtensions().getExtraProperties().set("versions", VersionProperties.getVersions());
     }
 
-    /**
-     * Makes dependencies non-transitive.
-     * <p>
-     * Gradle allows setting all dependencies as non-transitive very easily.
-     * Sadly this mechanism does not translate into maven pom generation. In order
-     * to effectively make the pom act as if it has no transitive dependencies,
-     * we must exclude each transitive dependency of each direct dependency.
-     * <p>
-     * Determining the transitive deps of a dependency which has been resolved as
-     * non-transitive is difficult because the process of resolving removes the
-     * transitive deps. To sidestep this issue, we create a configuration per
-     * direct dependency version. This specially named and unique configuration
-     * will contain all of the transitive dependencies of this particular
-     * dependency. We can then use this configuration during pom generation
-     * to iterate the transitive dependencies and add excludes.
-     */
-    public static void configureConfigurations(Project project) {
-        // we are not shipping these jars, we act like dumb consumers of these things
-        if (project.getPath().startsWith(":test:fixtures") || project.getPath().equals(":build-tools")) {
-            return;
-        }
-        // fail on any conflicting dependency versions
-        project.getConfigurations().all(configuration -> {
-            if (configuration.getName().endsWith("Fixture")) {
-                // just a self contained test-fixture configuration, likely transitive and hellacious
-                return;
-            }
-            configuration.resolutionStrategy(ResolutionStrategy::failOnVersionConflict);
-        });
-
-        // disable transitive dependency management
-        SourceSetContainer sourceSets = project.getExtensions().getByType(SourceSetContainer.class);
-        sourceSets.all(sourceSet -> disableTransitiveDependenciesForSourceSet(project, sourceSet));
-    }
-
-    private static void disableTransitiveDependenciesForSourceSet(Project project, SourceSet sourceSet) {
-        List<String> sourceSetConfigurationNames = List.of(
-            sourceSet.getApiConfigurationName(),
-            sourceSet.getImplementationConfigurationName(),
-            sourceSet.getCompileOnlyConfigurationName(),
-            sourceSet.getRuntimeOnlyConfigurationName()
-        );
-
-        project.getConfigurations()
-            .matching(c -> sourceSetConfigurationNames.contains(c.getName()))
-            .configureEach(GradleUtils::disableTransitiveDependencies);
-    }
-
     /**
      * Adds compiler settings to the project
      */