Merge remote-tracking branch 'upstream/main' into dlm-tiny-retentions

* upstream/main: (43 commits)
  Unmute testAckedIndexing to see if it still fails on main (elastic#134682)
  Silence time zone ID deprecation warning for JDK 25 due to log4j2 bug. (elastic#134719)
  Adding a getUnmodifiableSourceAndMetadata() method to IngestDocument (elastic#134816)
  Mark the create-index-from-source action as publicly available on Serverless (elastic#134953)
  ESQL: Rename command from INLINESTATS to INLINE STATS (elastic#134827)
  Document multi index query support for simplified retrievers (elastic#134980)
  [ML] Fix YAMl test to use correct query parameter type (elastic#134999)
  [Transform] Wait for PIT to close (elastic#134955)
  Add XPath to XmlUtils (elastic#134923)
  Fixing conditional processor mutability bugs (elastic#134936)
  [Transform] Lower loglevel of 3 transform-related error messages from ERROR to WARN (elastic#134985)
  Unmute pattern text tests. (elastic#134981)
  Integrate weights into simplified RRF retriever syntax (elastic#132680)
  Mute org.elasticsearch.xpack.esql.qa.mixed.MixedClusterEsqlSpecIT test {csv-spec:stats.CountDistinctWithConditions} elastic#134993
  Update periodic java-ea build to test java 26 pre-release (elastic#134983)
  Mute org.elasticsearch.xpack.esql.ccq.MultiClusterSpecIT test {csv-spec:stats.CountDistinctWithConditions} elastic#134984
  Fix and unmute testIndexSettingProviderPrivateSetting (elastic#134861)
  Add missing common cat params (elastic#134870)
  Support querying multiple indices with the simplified RRF retriever (elastic#134822)
  Allow including semantic field embeddings in _source (elastic#134717)
  ...

Author: szybia

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.8", "8.19.4", "9.1.4", "9.2.0"]
+            BWC_VERSION: ["8.18.8", "8.19.4", "9.0.8", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-java-ea.template.yml

@@ -1,5 +1,5 @@
 env:
-  JAVA_EA_VERSION: "${JAVA_EA_VERSION:-25-pre}"
+  JAVA_EA_VERSION: "${JAVA_EA_VERSION:-26-pre}"
 
 steps:
   - group: bwc

.buildkite/pipelines/periodic-java-ea.yml

@@ -1,6 +1,6 @@
 # This file is auto-generated. See .buildkite/pipelines/periodic-java-ea.template.yml
 env:
-  JAVA_EA_VERSION: "${JAVA_EA_VERSION:-25-pre}"
+  JAVA_EA_VERSION: "${JAVA_EA_VERSION:-26-pre}"
 
 steps:
   - group: bwc
@@ -385,8 +385,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.7 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.0.7#bwcTest
+      - label: 9.0.8 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.0.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -395,7 +395,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.7
+          BWC_VERSION: 9.0.8
         retry:
           automatic:
             - exit_status: "-1"
@@ -519,7 +519,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.8", "8.19.4", "9.1.4", "9.2.0"]
+            BWC_VERSION: ["8.18.8", "8.19.4", "9.0.8", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -350,8 +350,8 @@ steps:
         env:
           BWC_VERSION: 8.19.4
 
-      - label: "{{matrix.image}} / 9.0.7 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.7
+      - label: "{{matrix.image}} / 9.0.8 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.8
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -364,7 +364,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.7
+          BWC_VERSION: 9.0.8
 
       - label: "{{matrix.image}} / 9.1.4 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.4

.buildkite/pipelines/periodic.yml

@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.7 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.7#bwcTest
+      - label: 9.0.8 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.7
+          BWC_VERSION: 9.0.8
         retry:
           automatic:
             - exit_status: "-1"
@@ -506,7 +506,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.18.8", "8.19.4", "9.1.4", "9.2.0"]
+            BWC_VERSION: ["8.18.8", "8.19.4", "9.0.8", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -550,7 +550,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.18.8", "8.19.4", "9.1.4", "9.2.0"]
+            BWC_VERSION: ["8.18.8", "8.19.4", "9.0.8", "9.1.4", "9.2.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pull-requests.json

@@ -33,7 +33,8 @@
       "retrigger_label_regex": "v[0-9]+\\.[0-9]+\\.[0-9]+",
       "cancel_intermediate_builds": true,
       "cancel_intermediate_builds_on_comment": false,
-      "skip_duplicate_builds": true
+      "skip_duplicate_builds": true,
+      "skip_ci_label": "skip-transport-version-generation"
     },
     {
       "enabled": true,

.buildkite/scripts/update-transport-versions.sh

@@ -31,8 +31,8 @@ fi
 
 .ci/scripts/run-gradle.sh generateTransportVersion --backport-branches="${backport_branches}"
 
-if git diff --exit-code; then
-  echo "No changes found after updating transport versions. Don't need to auto commit."
+if git diff --quiet -- "server/src/main/resources/transport/**"; then
+  echo "No transport version changes found after update. Skipping auto commit."
   exit 0
 fi
 

.ci/bwcVersions

@@ -19,6 +19,6 @@ BWC_VERSION:
   - "8.17.10"
   - "8.18.8"
   - "8.19.4"
-  - "9.0.7"
+  - "9.0.8"
   - "9.1.4"
   - "9.2.0"

.ci/snapshotBwcVersions

@@ -1,5 +1,6 @@
 BWC_VERSION:
   - "8.18.8"
   - "8.19.4"
+  - "9.0.8"
   - "9.1.4"
   - "9.2.0"

build-tools-internal/src/integTest/groovy/org/elasticsearch/gradle/internal/transport/AbstractTransportVersionFuncTest.groovy

@@ -126,8 +126,9 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
                 currentUpperBoundName = '9.2'
             }
         """
-        referableTransportVersion("existing_91", "8012000")
-        referableTransportVersion("existing_92", "8123000,8012001")
+        referableAndReferencedTransportVersion("existing_91", "8012000")
+        referableAndReferencedTransportVersion("older_92", "8122000")
+        referableAndReferencedTransportVersion("existing_92", "8123000,8012001")
         unreferableTransportVersion("initial_9.0.0", "8000000")
         unreferableTransportVersion("initial_8.19.7", "7123001")
         transportVersionUpperBound("9.2", "existing_92", "8123000")
@@ -140,10 +141,6 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
                 return null;
             }
         """)
-        javaSource("myserver", "org.elasticsearch", "Dummy", "", """
-            static final TransportVersion existing91 = TransportVersion.fromName("existing_91");
-            static final TransportVersion existing92 = TransportVersion.fromName("existing_92");
-        """)
 
         file("myplugin/build.gradle") << """
             apply plugin: 'java-library'
@@ -155,7 +152,11 @@ class AbstractTransportVersionFuncTest extends AbstractGradleFuncTest {
         """
 
         setupLocalGitRepo()
-        execute("git checkout -b main")
+        String currentBranch = execute("git branch --show-current")
+        if (currentBranch.strip().equals("main") == false) {
+            // make sure a main branch exists, some CI doesn't have main set as the default branch
+            execute("git checkout -b main")
+        }
         execute("git checkout -b test")
     }