Merge branch 'main' into esql-ccs-simplified-grammar

Author: pawankartik-elastic

.buildkite/hooks/pre-command

@@ -94,6 +94,18 @@ if [[ "${USE_PROD_DOCKER_CREDENTIALS:-}" == "true" ]]; then
   fi
 fi
 
+if [[ "${USE_PERF_CREDENTIALS:-}" == "true" ]]; then
+  PERF_METRICS_HOST=$(vault read -field=es_host /secret/ci/elastic-elasticsearch/esbench-metics)
+  PERF_METRICS_INDEX="dummy-micro-benchmarks"
+  PERF_METRICS_USERNAME=$(vault read -field=es_username /secret/ci/elastic-elasticsearch/esbench-metics)
+  PERF_METRICS_PASSWORD=$(vault read -field=es_password /secret/ci/elastic-elasticsearch/esbench-metics)
+
+  export PERF_METRICS_HOST
+  export PERF_METRICS_INDEX
+  export PERF_METRICS_USERNAME
+  export PERF_METRICS_PASSWORD
+fi
+
 # Authenticate to the Docker Hub public read-only registry
 if which docker > /dev/null 2>&1; then
   DOCKERHUB_REGISTRY_USERNAME="$(vault read -field=username secret/ci/elastic-elasticsearch/docker_hub_public_ro_credentials)"

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
+            BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-micro-benchmarks.yml

@@ -0,0 +1,13 @@
+steps:
+  - label: periodic-micro-benchmarks
+    command: |
+      .ci/scripts/run-gradle.sh :benchmarks:run --args 'org.elasticsearch.benchmark._nightly -rf json -rff build/result.json'
+      .buildkite/scripts/index-micro-benchmark-results.sh
+    env:
+      USE_PERF_CREDENTIALS: "true"
+    timeout_in_minutes: 300
+    agents:
+      provider: gcp
+      image: family/elasticsearch-ubuntu-2004
+      machineType: custom-32-98304
+      buildDirectory: /dev/shm/bk

.buildkite/pipelines/periodic-packaging.yml

@@ -303,8 +303,8 @@ steps:
         env:
           BWC_VERSION: 8.16.6
 
-      - label: "{{matrix.image}} / 8.17.7 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.7
+      - label: "{{matrix.image}} / 8.17.8 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.8
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -317,10 +317,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.7
+          BWC_VERSION: 8.17.8
 
-      - label: "{{matrix.image}} / 8.18.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.2
+      - label: "{{matrix.image}} / 8.18.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -333,7 +333,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.2
+          BWC_VERSION: 8.18.3
 
       - label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
@@ -351,8 +351,8 @@ steps:
         env:
           BWC_VERSION: 8.19.0
 
-      - label: "{{matrix.image}} / 9.0.2 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.2
+      - label: "{{matrix.image}} / 9.0.3 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.3
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -365,7 +365,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.2
+          BWC_VERSION: 9.0.3
 
       - label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0

.buildkite/pipelines/periodic.yml

@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.7 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.7#bwcTest
+      - label: 8.17.8 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.7
+          BWC_VERSION: 8.17.8
         retry:
           automatic:
             - exit_status: "-1"
@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.2#bwcTest
+      - label: 8.18.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.2
+          BWC_VERSION: 8.18.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.2 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.2#bwcTest
+      - label: 9.0.3 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.3#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.2
+          BWC_VERSION: 9.0.3
         retry:
           automatic:
             - exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
+            BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -529,7 +529,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.17.7", "8.18.2", "8.19.0", "9.0.2", "9.1.0"]
+            BWC_VERSION: ["8.17.8", "8.18.3", "8.19.0", "9.0.3", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/scripts/index-micro-benchmark-results.sh

@@ -0,0 +1,10 @@
+#!/bin/bash
+
+jq -c '.[]' "benchmarks/build/result.json" | while read -r doc; do
+  doc=$(echo "$doc" | jq --argjson timestamp "$(date +%s000)" '. + {"@timestamp": $timestamp}')
+  echo "Indexing $(echo "$doc" | jq -r '.benchmark')"
+  curl -s -X POST "https://$PERF_METRICS_HOST/$PERF_METRICS_INDEX/_doc" \
+    -u "$PERF_METRICS_USERNAME:$PERF_METRICS_PASSWORD" \
+    -H 'Content-Type: application/json' \
+    -d "$doc"
+done

.ci/bwcVersions

@@ -16,8 +16,8 @@ BWC_VERSION:
   - "8.14.3"
   - "8.15.5"
   - "8.16.6"
-  - "8.17.7"
-  - "8.18.2"
+  - "8.17.8"
+  - "8.18.3"
   - "8.19.0"
-  - "9.0.2"
+  - "9.0.3"
   - "9.1.0"

.ci/scripts/run-gradle.sh

@@ -41,4 +41,4 @@ else
 fi
 
 set -e
-$GRADLEW -S --max-workers=$MAX_WORKERS $@
+$GRADLEW -S --max-workers=$MAX_WORKERS "$@"

.ci/snapshotBwcVersions

@@ -1,6 +1,6 @@
 BWC_VERSION:
-  - "8.17.7"
-  - "8.18.2"
+  - "8.17.8"
+  - "8.18.3"
   - "8.19.0"
-  - "9.0.2"
+  - "9.0.3"
   - "9.1.0"

benchmarks/src/main/java/org/elasticsearch/benchmark/esql/QueryPlanningBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.esql;
+package org.elasticsearch.benchmark._nightly.esql;
 
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.common.settings.Settings;
@@ -67,9 +67,9 @@ public class QueryPlanningBenchmark {
     }
 
     private PlanTelemetry telemetry;
-    private EsqlParser parser;
-    private Analyzer analyzer;
-    private LogicalPlanOptimizer optimizer;
+    private EsqlParser defaultParser;
+    private Analyzer manyFieldsAnalyzer;
+    private LogicalPlanOptimizer defaultOptimizer;
 
     @Setup
     public void setup() {
@@ -100,8 +100,8 @@ public void setup() {
         var functionRegistry = new EsqlFunctionRegistry();
 
         telemetry = new PlanTelemetry(functionRegistry);
-        parser = new EsqlParser();
-        analyzer = new Analyzer(
+        defaultParser = new EsqlParser();
+        manyFieldsAnalyzer = new Analyzer(
             new AnalyzerContext(
                 config,
                 functionRegistry,
@@ -112,10 +112,10 @@ public void setup() {
             ),
             new Verifier(new Metrics(functionRegistry), new XPackLicenseState(() -> 0L))
         );
-        optimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
+        defaultOptimizer = new LogicalPlanOptimizer(new LogicalOptimizerContext(config, FoldContext.small()));
     }
 
-    private LogicalPlan plan(String query) {
+    private LogicalPlan plan(EsqlParser parser, Analyzer analyzer, LogicalPlanOptimizer optimizer, String query) {
         var parsed = parser.createStatement(query, new QueryParams(), telemetry);
         var analyzed = analyzer.analyze(parsed);
         var optimized = optimizer.optimize(analyzed);
@@ -124,6 +124,6 @@ private LogicalPlan plan(String query) {
 
     @Benchmark
     public void manyFields(Blackhole blackhole) {
-        blackhole.consume(plan("FROM test | LIMIT 10"));
+        blackhole.consume(plan(defaultParser, manyFieldsAnalyzer, defaultOptimizer, "FROM test | LIMIT 10"));
     }
 }