Merge branch 'main' into update-gradle-wrapper-920

Author: breskeby

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.9", "8.19.6", "9.0.9", "9.1.6", "9.2.0", "9.3.0"]
+            BWC_VERSION: ["8.19.6", "9.1.6", "9.2.0", "9.3.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-java-ea.yml

@@ -347,8 +347,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.9 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.18.9#bwcTest
+      - label: 8.18.8 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.18.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -357,7 +357,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.9
+          BWC_VERSION: 8.18.8
         retry:
           automatic:
             - exit_status: "-1"
@@ -385,8 +385,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.9 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.0.9#bwcTest
+      - label: 9.0.8 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.0.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -395,7 +395,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.9
+          BWC_VERSION: 9.0.8
         retry:
           automatic:
             - exit_status: "-1"
@@ -538,7 +538,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.9", "8.19.6", "9.0.9", "9.1.6", "9.2.0", "9.3.0"]
+            BWC_VERSION: ["8.19.6", "9.1.6", "9.2.0", "9.3.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -318,8 +318,8 @@ steps:
         env:
           BWC_VERSION: 8.17.10
 
-      - label: "{{matrix.image}} / 8.18.9 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.9
+      - label: "{{matrix.image}} / 8.18.8 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.8
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -332,7 +332,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.9
+          BWC_VERSION: 8.18.8
 
       - label: "{{matrix.image}} / 8.19.6 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.6
@@ -350,8 +350,8 @@ steps:
         env:
           BWC_VERSION: 8.19.6
 
-      - label: "{{matrix.image}} / 9.0.9 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.9
+      - label: "{{matrix.image}} / 9.0.8 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.8
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -364,7 +364,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.9
+          BWC_VERSION: 9.0.8
 
       - label: "{{matrix.image}} / 9.1.6 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.6

.buildkite/pipelines/periodic.yml

@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.9 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.9#bwcTest
+      - label: 8.18.8 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.9
+          BWC_VERSION: 8.18.8
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.9 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.9#bwcTest
+      - label: 9.0.8 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.9
+          BWC_VERSION: 9.0.8
         retry:
           automatic:
             - exit_status: "-1"
@@ -525,7 +525,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.18.9", "8.19.6", "9.0.9", "9.1.6", "9.2.0", "9.3.0"]
+            BWC_VERSION: ["8.19.6", "9.1.6", "9.2.0", "9.3.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -569,7 +569,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.18.9", "8.19.6", "9.0.9", "9.1.6", "9.2.0", "9.3.0"]
+            BWC_VERSION: ["8.19.6", "9.1.6", "9.2.0", "9.3.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/scripts/trigger-if-java-ea-new-build.sh

@@ -13,7 +13,7 @@
 RECENT_TIME_WINDOW=${RECENT_TIME_WINDOW:-24}
 
 # Extract current JDK major version from bundled_jdk in version.properties
-CURRENT_JDK=$(grep "^bundled_jdk =" build-tools-internal/version.properties | cut -d'=' -f2 | tr -d ' ' | cut -d'.' -f1)
+CURRENT_JDK=$(grep "^bundled_jdk =" build-tools-internal/version.properties | cut -d'=' -f2 | tr -d ' ' | cut -d'+' -f1)
 TARGET_JDK=$((CURRENT_JDK + 1))
 
 echo "Current JDK major version: $CURRENT_JDK"
@@ -80,13 +80,13 @@ echo "SHOULD_TRIGGER: $SHOULD_TRIGGER"
 
 if [[ "$SHOULD_TRIGGER" == "true" ]]; then
   EFFECTIVE_START_DATE=$(date -u -d "@$BUILD_TIME" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || date -u -r "$BUILD_TIME" +"%Y-%m-%dT%H:%M:%SZ" 2>/dev/null || echo "")
-  
+
   # Use "master" branch for elasticsearch-performance-esbench-jdk when BUILDKITE_BRANCH is "main"
   TRIGGER_BRANCH="$BUILDKITE_BRANCH"
   if [[ "$BUILDKITE_BRANCH" == "main" ]]; then
     TRIGGER_BRANCH="master"
   fi
-  
+
   echo "Triggering performance-esbench-jdk for new jdk build $JDK_IDENTIFIER"
   cat << EOF | buildkite-agent pipeline upload
 steps:
@@ -98,5 +98,7 @@ steps:
     env:
       EFFECTIVE_START_DATE: "$EFFECTIVE_START_DATE"
       EXECUTION_MODE: "start-run"
+      JDK_VERSION: "$TARGET_JDK"
+      JDK_IDENTIFIER: "$JDK_IDENTIFIER"
 EOF
 fi

.ci/bwcVersions

@@ -17,9 +17,9 @@ BWC_VERSION:
   - "8.15.5"
   - "8.16.6"
   - "8.17.10"
-  - "8.18.9"
+  - "8.18.8"
   - "8.19.6"
-  - "9.0.9"
+  - "9.0.8"
   - "9.1.6"
   - "9.2.0"
   - "9.3.0"

.ci/snapshotBwcVersions

@@ -1,7 +1,5 @@
 BWC_VERSION:
-  - "8.18.9"
   - "8.19.6"
-  - "9.0.9"
   - "9.1.6"
   - "9.2.0"
   - "9.3.0"

.swp

@@ -9,14 +9,12 @@
 
 package org.elasticsearch.benchmark._nightly.esql;
 
+import org.elasticsearch.TransportVersion;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
 import org.elasticsearch.index.IndexMode;
 import org.elasticsearch.license.XPackLicenseState;
-import org.elasticsearch.xpack.esql.analysis.Analyzer;
-import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
-import org.elasticsearch.xpack.esql.analysis.EnrichResolution;
-import org.elasticsearch.xpack.esql.analysis.Verifier;
+import org.elasticsearch.xpack.esql.analysis.*;
 import org.elasticsearch.xpack.esql.core.expression.FoldContext;
 import org.elasticsearch.xpack.esql.core.type.EsField;
 import org.elasticsearch.xpack.esql.core.util.DateUtils;
@@ -29,7 +27,6 @@
 import org.elasticsearch.xpack.esql.parser.EsqlParser;
 import org.elasticsearch.xpack.esql.parser.QueryParams;
 import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan;
-import org.elasticsearch.xpack.esql.plugin.EsqlPlugin;
 import org.elasticsearch.xpack.esql.plugin.QueryPragmas;
 import org.elasticsearch.xpack.esql.session.Configuration;
 import org.elasticsearch.xpack.esql.telemetry.Metrics;
@@ -79,16 +76,16 @@ public void setup() {
             Locale.US,
             null,
             null,
-            new QueryPragmas(Settings.EMPTY),
-            EsqlPlugin.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
-            EsqlPlugin.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
+            QueryPragmas.EMPTY,
+            AnalyzerSettings.QUERY_RESULT_TRUNCATION_MAX_SIZE.getDefault(Settings.EMPTY),
+            AnalyzerSettings.QUERY_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
             "",
             false,
             Map.of(),
             System.nanoTime(),
             false,
-            EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
-            EsqlPlugin.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY)
+            AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.getDefault(Settings.EMPTY),
+            AnalyzerSettings.QUERY_TIMESERIES_RESULT_TRUNCATION_DEFAULT_SIZE.get(Settings.EMPTY)
         );
 
         var fields = 10_000;
@@ -102,6 +99,9 @@ public void setup() {
 
         var functionRegistry = new EsqlFunctionRegistry();
 
+        // Assume all nodes are on the current version for the benchmark.
+        TransportVersion minimumVersion = TransportVersion.current();
+
         telemetry = new PlanTelemetry(functionRegistry);
         defaultParser = new EsqlParser();
         manyFieldsAnalyzer = new Analyzer(
@@ -111,11 +111,12 @@ public void setup() {
                 IndexResolution.valid(esIndex),
                 Map.of(),
                 new EnrichResolution(),
-                InferenceResolution.EMPTY
+                InferenceResolution.EMPTY,
+                minimumVersion
             ),
             new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
         );
-        defaultOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
+        defaultOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small(), minimumVersion));
     }
 
     private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -369,7 +369,7 @@ public void benchmark() {
             blockFactory,
             ByteSizeValue.ofMb(1).getBytes(),
             fields(name),
-            new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, () -> {
+            new IndexedByShardIdFromSingleton<>(new ValuesSourceReaderOperator.ShardContext(reader, (sourcePaths) -> {
                 throw new UnsupportedOperationException("can't load _source here");
             }, EsqlPlugin.STORED_FIELDS_SEQUENTIAL_PROPORTION.getDefault(Settings.EMPTY))),
             0