Merge branch 'main' into ts-in-generative-tests

Author: limotova

.buildkite/pipelines/pull-request-transport-versions.yml

@@ -1 +1,10 @@
-steps: []
+steps:
+  - label: update-transport-versions
+    command: '.buildkite/scripts/update-transport-versions.sh'
+    # New transport versions are always added via the main branch
+    if: build.env('BUILDKITE_PULL_REQUEST_BASE_BRANCH') == 'main'
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2404
+      machineType: custom-16-32768
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/pull-request/validate-transport-version-backport.yml

@@ -0,0 +1,5 @@
+config:
+  skip-target-branches: "main"
+steps:
+  - label: validate-transport-version-backport
+    command: .buildkite/scripts/validate-transport-version-backport.sh

.buildkite/scripts/pull-request/__snapshots__/pipeline.test.ts.snap

@@ -309,3 +309,68 @@ exports[`generatePipelines should generate correct pipelines with a non-docs cha
   },
 ]
 `;
+
+exports[`generatePipelines should generate correct pipelines with a different branch that is not skipped 1`] = `
+[
+  {
+    "name": "bwc-snapshots",
+    "pipeline": {
+      "steps": [
+        {
+          "group": "bwc-snapshots",
+          "steps": [
+            {
+              "agents": {
+                "buildDirectory": "/dev/shm/bk",
+                "image": "family/elasticsearch-ubuntu-2404",
+                "machineType": "custom-32-98304",
+                "provider": "gcp",
+              },
+              "command": ".ci/scripts/run-gradle.sh -Dignore.tests.seed v{{matrix.BWC_VERSION}}#bwcTest",
+              "env": {
+                "BWC_VERSION": "{{matrix.BWC_VERSION}}",
+              },
+              "label": "{{matrix.BWC_VERSION}} / bwc-snapshots",
+              "matrix": {
+                "setup": {
+                  "BWC_VERSION": [
+                    "7.17.14",
+                    "8.10.3",
+                    "8.11.0",
+                  ],
+                },
+              },
+              "timeout_in_minutes": 300,
+            },
+          ],
+        },
+      ],
+    },
+  },
+  {
+    "name": "skip-branch",
+    "pipeline": {
+      "steps": [
+        {
+          "command": "echo 1",
+          "label": "skip-me",
+        },
+      ],
+    },
+  },
+  {
+    "name": "using-defaults",
+    "pipeline": {
+      "env": {
+        "CUSTOM_ENV_VAR": "value",
+      },
+      "steps": [
+        {
+          "command": "echo 'hello world'",
+          "label": "test-step",
+        },
+      ],
+    },
+  },
+]
+`;

.buildkite/scripts/pull-request/mocks/pipelines/skip-branch.yml

@@ -0,0 +1,5 @@
+config:
+  skip-target-branches: "test-branch"
+steps:
+  - label: skip-me
+    command: echo 1

.buildkite/scripts/pull-request/pipeline.test.ts

@@ -8,6 +8,7 @@ describe("generatePipelines", () => {
     setBwcVersionsPath(`${import.meta.dir}/mocks/bwcVersions`);
     setSnapshotBwcVersionsPath(`${import.meta.dir}/mocks/snapshotBwcVersions`);
 
+    process.env["GITHUB_PR_TARGET_BRANCH"] = "test-branch";
     process.env["GITHUB_PR_LABELS"] = "test-label-1,test-label-2";
     process.env["GITHUB_PR_TRIGGER_COMMENT"] = "";
   });
@@ -36,6 +37,12 @@ describe("generatePipelines", () => {
     testWithTriggerCheck(`${import.meta.dir}/mocks/pipelines`, ["build.gradle"]);
   });
 
+  test("should generate correct pipelines with a different branch that is not skipped", () => {
+    process.env["GITHUB_PR_TARGET_BRANCH"] = "main";
+
+    testWithTriggerCheck(`${import.meta.dir}/mocks/pipelines`, ["build.gradle"]);
+  });
+
   test("should generate correct pipeline when using a trigger comment for it", () => {
     process.env["GITHUB_PR_TRIGGER_COMMENT"] = "run elasticsearch-ci/using-defaults";
 

.buildkite/scripts/pull-request/pipeline.ts

@@ -50,6 +50,14 @@ const changedFilesIncludedCheck = (pipeline: EsPipeline, changedFiles: string[])
   return true;
 };
 
+const checkTargetBranch = (pipeline: EsPipeline, targetBranch: string | undefined) => {
+  if (!targetBranch || !pipeline.config?.["skip-target-branches"]) {
+    return true;
+  }
+
+  return !getArray(pipeline.config["skip-target-branches"]).some((branch) => branch === targetBranch);
+};
+
 const triggerCommentCheck = (pipeline: EsPipeline): boolean => {
   if (process.env["GITHUB_PR_TRIGGER_COMMENT"] && pipeline.config?.["trigger-phrase"]) {
     return !!process.env["GITHUB_PR_TRIGGER_COMMENT"].match(pipeline.config["trigger-phrase"]);
@@ -138,6 +146,7 @@ export const generatePipelines = (
   }
 
   let filters: ((pipeline: EsPipeline) => boolean)[] = [
+    (pipeline) => checkTargetBranch(pipeline, process.env["GITHUB_PR_TARGET_BRANCH"]),
     (pipeline) => labelCheckAllow(pipeline, labels),
     (pipeline) => labelCheckSkip(pipeline, labels),
     (pipeline) => changedFilesExcludedCheck(pipeline, changedFiles),

.buildkite/scripts/pull-request/types.ts

@@ -5,6 +5,7 @@ export type EsPipelineConfig = {
     "included-regions"?: string | string[];
     "excluded-regions"?: string | string[];
     "trigger-phrase"?: string;
+    "skip-target-branches"?: string | string[];
   };
 };
 

.buildkite/scripts/update-transport-versions.sh

@@ -0,0 +1,51 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ -z "${BUILDKITE_PULL_REQUEST:-}" ]]; then
+  echo "Not a pull request, skipping transport version update"
+  exit 0
+fi
+
+if ! git diff --exit-code; then
+  echo "Changes are present before updating transport versions, not running"
+  git status
+  exit 0
+fi
+
+NEW_COMMIT_MESSAGE="[CI] Update transport version definitions"
+
+echo "--- Generating updated transport version definitions"
+# Calculate backport branches based on pull request version labels
+backport_branches=$(
+  echo "${GITHUB_PR_LABELS}" \
+    | tr ',' '\n' \
+    | grep -E "v[0-9]+\.[0-9]+\.[0-9]+" \
+    | sed -E 's/^v([0-9]+)\.([0-9]+)\.[0-9]+$/\1.\2/' \
+    | paste -sd, -
+)
+
+if [[ -z "${backport_branches}" ]]; then
+  echo "Skipping as pull request contains no version labels"
+  exit 0
+fi
+
+.ci/scripts/run-gradle.sh generateTransportVersion --backport-branches="${backport_branches}"
+
+if git diff --exit-code; then
+  echo "No changes found after updating transport versions. Don't need to auto commit."
+  exit 0
+fi
+
+git config --global user.name elasticsearchmachine
+git config --global user.email '[email protected]'
+
+gh pr checkout "${BUILDKITE_PULL_REQUEST}"
+git add -A .
+git commit -m "$NEW_COMMIT_MESSAGE"
+git push
+
+# After the git push, the new commit will trigger a new build within a few seconds and this build should get cancelled
+# So, let's just sleep to give the build time to cancel itself without an error
+# If it doesn't get cancelled for some reason, then exit with an error, because we don't want this build to be green (we just don't want it to generate an error either)
+sleep 300
+exit 1

.buildkite/scripts/validate-transport-version-backport.sh

@@ -0,0 +1,30 @@
+#!/bin/bash
+set -euo pipefail
+
+if [[ "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" == "main" ]]; then
+  # Don't run on PRs targeting main
+  exit 0
+fi
+
+echo "--- Looking for transport version changes"
+
+# Get any changes in this pull request to transport definitions
+git fetch origin "${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" --quiet
+changed_files=$(git diff --name-only "origin/${BUILDKITE_PULL_REQUEST_BASE_BRANCH}" | grep -E "server/src/main/resources/transport/definitions/.*\.csv" || true)
+
+if [[ -z "${changed_files}" ]]; then
+  echo "No transport version changes detected."
+  exit 0
+fi
+
+# Compare those files against the main branch to ensure they are the same
+git fetch origin main --quiet
+while IFS= read -r file; do
+  if ! git diff --quiet origin/main -- "${file}"; then
+      echo "Changes to transport definition [${file}] missing from main branch."
+      echo "Transport changes must first be merged to main before being backported."
+      exit 1
+  fi
+done <<< "${changed_files}"
+
+echo "All transport changes exist in main branch."

BUILDING.md

@@ -97,6 +97,70 @@ will have the `origin` attribute been set to `Generated by Gradle`.
 > If you want to add a level of verification you can manually confirm the checksum (e.g. by looking it up on the website of the library)
 > Please replace the content of the `origin` attribute by `official site` in that case.
 
+##### Handling transitive dependencies
+
+Dependency management is a critical aspect of maintaining a secure and reliable build system, requiring explicit control over what we rely on. The Elasticsearch build mainly uses component metadata rules declared in the `ComponentMetadataRulesPlugin`
+plugin to manage transitive dependencies and avoid version conflicts.
+This approach ensures we have explicit control over all dependencies used in the build.
+
+###### General Guidelines
+
+1. **Avoid unused transitive dependencies** - Dependencies that are not actually used by our code should be excluded to reduce the attack surface and avoid potential conflicts.
+
+2. **Prefer versions declared in `build-tools-internal/version.properties`** - All dependency versions should be centrally managed in this file to ensure consistency across the entire build.
+
+3. **Libraries required to compile our code should be direct dependencies** - If we directly use a library in our source code, it should be declared as a direct dependency rather than relying on it being transitively available.
+
+###### Component Metadata Rules
+
+We use two main types of component metadata rules at this point to manage transitive dependencies:
+
+- **`ExcludeAllTransitivesRule`** - Excludes all transitive dependencies for libraries where we want complete control over dependencies or the transitive dependencies are unused.
+
+- **`ExcludeOtherGroupsTransitiveRule`** - Excludes transitive dependencies that don't belong to the same group as the direct dependency, while keeping same-group dependencies.
+-
+- **`ExcludeByGroup`** - Excludes transitive dependencies that match a specific groupId while keeping all other transitive dependencies with different groupIds.
+
+Examples from the `ComponentMetadataRulesPlugin`:
+
+```gradle
+// Exclude all transitives - used when transitive deps are unused or problematic
+components.withModule("com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", ExcludeAllTransitivesRule.class);
+
+// Exclude other groups - used when we want same-group deps but not external ones
+components.withModule("com.azure:azure-core", ExcludeOtherGroupsTransitiveRule.class);
+
+// Exclude only specific groups - used when we want exclude specific group of transitive deps.
+components.withModule("org.apache.logging.log4j:log4j-api", ExcludeByGroup.class, rule -> {
+    rule.params(List.of("biz.aQute.bnd", "org.osgi"));
+});
+```
+
+###### Common Scenarios
+
+**Version Conflicts**: When a transitive dependency brings in a different version than what we use:
+```gradle
+// brings in jackson-databind and jackson-annotations not used
+components.withModule("com.fasterxml.jackson.dataformat:jackson-dataformat-cbor", ExcludeAllTransitivesRule.class);
+```
+
+**Unused Dependencies**: When transitive dependencies are not actually used:
+```gradle
+// brings in azure-core-http-netty. not used
+components.withModule("com.azure:azure-core-http-netty", ExcludeAllTransitivesRule.class);
+```
+
+**Mismatching Version Dependencies**: When other versions are required:
+```gradle
+// brings in org.slf4j:slf4j-api:1.7.25. We use 2.0.6
+components.withModule("org.apache.directory.api:api-asn1-ber", ExcludeOtherGroupsTransitiveRule.class);
+```
+
+When adding or updating dependencies, ensure that any required transitive dependencies are either:
+1. Already available as direct dependencies with compatible versions
+2. Added as direct dependencies if they're actually used by our code
+3. Properly excluded if they're not needed
+
 #### Custom plugin and task implementations
 
 Build logic that is used across multiple subprojects should be considered to be moved into a Gradle plugin with according Gradle task implementation.