Merge branch 'main' into Add-IBM-Granite-support-for-completion-and-chat-completion

# Conflicts:
#	server/src/main/java/org/elasticsearch/TransportVersions.java
#	x-pack/plugin/inference/qa/inference-service-tests/src/javaRestTest/java/org/elasticsearch/xpack/inference/InferenceGetServicesIT.java

Author: Evgenii-Kazannik

.backportrc.json

@@ -1,9 +1,9 @@
 {
   "upstream" : "elastic/elasticsearch",
-  "targetBranchChoices" : [ "main", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
+  "targetBranchChoices" : [ "main", "9.1", "9.0", "8.19", "8.18", "8.17", "8.16", "8.15", "8.14", "8.13", "8.12", "8.11", "8.10", "8.9", "8.8", "8.7", "8.6", "8.5", "8.4", "8.3", "8.2", "8.1", "8.0", "7.17", "6.8" ],
   "targetPRLabels" : [ "backport" ],
   "branchLabelMapping" : {
-    "^v9.1.0$" : "main",
+    "^v9.2.0$" : "main",
     "^v(\\d+).(\\d+).\\d+(?:-(?:alpha|beta|rc)\\d+)?$" : "$1.$2"
   }
 }

.buildkite/hooks/pre-command

@@ -64,6 +64,14 @@ if [[ "${USE_LUCENE_SNAPSHOT_CREDS:-}" == "true" ]]; then
   unset data
 fi
 
+if [[ "${USE_MAVEN_GPG:-}" == "true" ]]; then
+  vault_path="kv/ci-shared/release-eng/team-release-secrets/es-delivery/gpg"
+  ORG_GRADLE_PROJECT_signingKey=$(vault kv get --field="private_key" $vault_path)
+  ORG_GRADLE_PROJECT_signingPassword=$(vault kv get --field="passphase" $vault_path)
+  export ORG_GRADLE_PROJECT_signingKey
+  export ORG_GRADLE_PROJECT_signingPassword
+fi
+
 if [[ "${USE_DRA_CREDENTIALS:-}" == "true" ]]; then
   DRA_VAULT_ROLE_ID_SECRET=$(vault read -field=role-id secret/ci/elastic-elasticsearch/legacy-vault-credentials)
   export DRA_VAULT_ROLE_ID_SECRET

.buildkite/pipelines/dra-workflow.yml

@@ -2,6 +2,7 @@ steps:
   - command: .buildkite/scripts/dra-workflow.sh
     env:
       USE_DRA_CREDENTIALS: "true"
+      USE_MAVEN_GPG: "true"
       USE_PROD_DOCKER_CREDENTIALS: "true"
     agents:
       provider: gcp

.buildkite/pipelines/intake.yml

@@ -57,14 +57,15 @@ steps:
       image: family/elasticsearch-ubuntu-2004
       machineType: n1-standard-32
       buildDirectory: /dev/shm/bk
+
   - group: bwc-snapshots
     steps:
       - label: "{{matrix.BWC_VERSION}} / bwc-snapshots"
         command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dorg.elasticsearch.build.cache.push=true -Dignore.tests.seed -Dscan.capture-file-fingerprints v$$BWC_VERSION#bwcTest
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
+            BWC_VERSION: ["8.17.9", "8.18.4", "8.19.0", "9.0.4", "9.1.0", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -74,9 +75,6 @@ steps:
           BWC_VERSION: "{{matrix.BWC_VERSION}}"
   - label: bc-upgrade
     command: ".buildkite/scripts/run-bc-upgrade-tests.sh"
-    agents:
-      image: "docker.elastic.co/ci-agent-images/eck-region/buildkite-agent:1.5"
-      memory: "4G"
   - group: lucene-compat
     steps:
       - label: "{{matrix.LUCENE_VERSION}} / lucene-compat"

.buildkite/pipelines/periodic-packaging.yml

@@ -302,8 +302,8 @@ steps:
         env:
           BWC_VERSION: 8.16.6
 
-      - label: "{{matrix.image}} / 8.17.8 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.8
+      - label: "{{matrix.image}} / 8.17.9 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.9
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -316,10 +316,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.8
+          BWC_VERSION: 8.17.9
 
-      - label: "{{matrix.image}} / 8.18.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.3
+      - label: "{{matrix.image}} / 8.18.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -332,7 +332,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.3
+          BWC_VERSION: 8.18.4
 
       - label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
@@ -350,8 +350,8 @@ steps:
         env:
           BWC_VERSION: 8.19.0
 
-      - label: "{{matrix.image}} / 9.0.3 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.3
+      - label: "{{matrix.image}} / 9.0.4 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.4
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -364,7 +364,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.3
+          BWC_VERSION: 9.0.4
 
       - label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0
@@ -382,6 +382,22 @@ steps:
         env:
           BWC_VERSION: 9.1.0
 
+      - label: "{{matrix.image}} / 9.2.0 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.2.0
+        timeout_in_minutes: 300
+        matrix:
+          setup:
+            image:
+              - rocky-8
+              - ubuntu-2404
+        agents:
+          provider: gcp
+          image: family/elasticsearch-{{matrix.image}}
+          machineType: custom-16-32768
+          buildDirectory: /dev/shm/bk
+        env:
+          BWC_VERSION: 9.2.0
+
   - group: packaging-tests-windows
     steps:
       - label: "{{matrix.image}} / packaging-tests-windows"

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.8 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.8#bwcTest
+      - label: 8.17.9 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.9#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.8
+          BWC_VERSION: 8.17.9
         retry:
           automatic:
             - exit_status: "-1"
@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.3#bwcTest
+      - label: 8.18.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.3
+          BWC_VERSION: 8.18.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.3 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.3#bwcTest
+      - label: 9.0.4 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.4#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.3
+          BWC_VERSION: 9.0.4
         retry:
           automatic:
             - exit_status: "-1"
@@ -420,6 +420,25 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
+      - label: 9.2.0 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.2.0#bwcTest
+        timeout_in_minutes: 300
+        agents:
+          provider: gcp
+          image: family/elasticsearch-ubuntu-2404
+          machineType: n1-standard-32
+          buildDirectory: /dev/shm/bk
+          preemptible: true
+        env:
+          BWC_VERSION: 9.2.0
+        retry:
+          automatic:
+            - exit_status: "-1"
+              limit: 3
+              signal_reason: none
+            - signal_reason: agent_stop
+              limit: 3
+
   - label: concurrent-search-tests
     command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true -Dtests.jvm.argline=-Des.concurrent_search=true -Des.concurrent_search=true functionalTests
     timeout_in_minutes: 420
@@ -487,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
+            BWC_VERSION: ["8.17.9", "8.18.4", "8.19.0", "9.0.4", "9.1.0", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -531,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
+            BWC_VERSION: ["8.17.9", "8.18.4", "8.19.0", "9.0.4", "9.1.0", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/scripts/run-bc-upgrade-tests.sh

@@ -24,7 +24,7 @@ select(.active_release == true) |
 (.build_candidates | to_entries | sort_by(.value.completed_at))) |
 last | .value.manifest_url")"
 
-if [[ -z "$MANIFEST_URL" ]]; then
+if [[ -z "$MANIFEST_URL" ]] || [[ "$MANIFEST_URL" == "null" ]]; then
    echo "No snapshots or build candidates for branch [$BUILDKITE_BRANCH]."
    echo "Skipping BC upgrade tests."
    exit 0

.buildkite/scripts/run-pr-upgrade-tests.sh

@@ -18,7 +18,8 @@ fi
 
 # Identify the merge base of the current commit (branch) and the base branch of the pull request.
 # PR upgrade tests are run from the merge base to the current commit.
-BASE_COMMIT=$(git merge-base $BUILDKITE_PULL_REQUEST_BASE_BRANCH $BUILDKITE_COMMIT)
+git fetch origin $BUILDKITE_PULL_REQUEST_BASE_BRANCH
+BASE_COMMIT=$(git merge-base origin/$BUILDKITE_PULL_REQUEST_BASE_BRANCH $BUILDKITE_COMMIT)
 
 VERSION=$(sed -n 's/^elasticsearch[[:space:]]*=[[:space:]]*\(.*\)/\1/p' build-tools-internal/version.properties)
 

.ci/bwcVersions

@@ -16,8 +16,9 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.5"
   - "8.16.6"
-  - "8.17.8"
-  - "8.18.3"
+  - "8.17.9"
+  - "8.18.4"
   - "8.19.0"
-  - "9.0.3"
+  - "9.0.4"
   - "9.1.0"
+  - "9.2.0"

.ci/scripts/resolve-dra-manifest.sh

@@ -6,7 +6,8 @@ strip_version() {
 }
 
 fetch_build() {
-  curl -sS https://artifacts-$1.elastic.co/$2/latest/$3.json \
+  >&2 echo "Checking for build id: https://artifacts-$1.elastic.co/$2/latest/$3.json"
+  curl -sSf https://artifacts-$1.elastic.co/$2/latest/$3.json \
     | jq -r '.build_id'
 }
 
@@ -15,7 +16,15 @@ BRANCH="${BRANCH:-$2}"
 ES_VERSION="${ES_VERSION:-$3}"
 WORKFLOW=${WORKFLOW:-$4}
 
-LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
+if [[ "$WORKFLOW" == "staging" ]]; then
+  LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $ES_VERSION)
+elif [[ "$WORKFLOW" == "snapshot" ]]; then
+  LATEST_BUILD=$(fetch_build $WORKFLOW $ARTIFACT $BRANCH)
+else
+  echo "Unknown workflow: $WORKFLOW"
+  exit 1
+fi
+
 LATEST_VERSION=$(strip_version $LATEST_BUILD)
 
 # If the latest artifact version doesn't match what we expect, try the corresponding version branch.