Merge branch 'main' into sagemaker

Author: prwhelan

.buildkite/pipelines/intake.yml

@@ -56,7 +56,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
+            BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/pipelines/periodic-packaging.yml

@@ -287,8 +287,8 @@ steps:
         env:
           BWC_VERSION: 8.15.5
 
-      - label: "{{matrix.image}} / 8.16.7 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.7
+      - label: "{{matrix.image}} / 8.16.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.16.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -301,10 +301,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.16.7
+          BWC_VERSION: 8.16.6
 
-      - label: "{{matrix.image}} / 8.17.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.5
+      - label: "{{matrix.image}} / 8.17.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.17.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -317,10 +317,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.17.5
+          BWC_VERSION: 8.17.6
 
-      - label: "{{matrix.image}} / 8.18.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.0
+      - label: "{{matrix.image}} / 8.18.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -333,7 +333,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.0
+          BWC_VERSION: 8.18.1
 
       - label: "{{matrix.image}} / 8.19.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.0
@@ -351,8 +351,8 @@ steps:
         env:
           BWC_VERSION: 8.19.0
 
-      - label: "{{matrix.image}} / 9.0.0 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.0
+      - label: "{{matrix.image}} / 9.0.1 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.1
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -365,7 +365,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.0
+          BWC_VERSION: 9.0.1
 
       - label: "{{matrix.image}} / 9.1.0 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.0

.buildkite/pipelines/periodic.yml

@@ -306,8 +306,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.16.7 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.7#bwcTest
+      - label: 8.16.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.16.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -316,7 +316,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.16.7
+          BWC_VERSION: 8.16.6
         retry:
           automatic:
             - exit_status: "-1"
@@ -325,8 +325,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.17.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.5#bwcTest
+      - label: 8.17.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.17.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -335,7 +335,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.17.5
+          BWC_VERSION: 8.17.6
         retry:
           automatic:
             - exit_status: "-1"
@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.0#bwcTest
+      - label: 8.18.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.0
+          BWC_VERSION: 8.18.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.0 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.0#bwcTest
+      - label: 9.0.1 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.1#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.0
+          BWC_VERSION: 9.0.1
         retry:
           automatic:
             - exit_status: "-1"
@@ -486,7 +486,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
+            BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004
@@ -533,7 +533,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.16.7", "8.17.5", "8.18.0", "8.19.0", "9.0.0", "9.1.0"]
+            BWC_VERSION: ["8.17.6", "8.18.1", "8.19.0", "9.0.1", "9.1.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2004

.buildkite/scripts/dra-workflow.trigger.sh

@@ -7,10 +7,6 @@ echo "steps:"
 source .buildkite/scripts/branches.sh
 
 for BRANCH in "${BRANCHES[@]}"; do
-  if [[ "$BRANCH" == "9.0" ]]; then
-    continue
-  fi
-
   INTAKE_PIPELINE_SLUG="elasticsearch-intake"
   BUILD_JSON=$(curl -sH "Authorization: Bearer ${BUILDKITE_API_TOKEN}" "https://api.buildkite.com/v2/organizations/elastic/pipelines/${INTAKE_PIPELINE_SLUG}/builds?branch=${BRANCH}&state=passed&per_page=1" | jq '.[0] | {commit: .commit, url: .web_url}')
   LAST_GOOD_COMMIT=$(echo "${BUILD_JSON}" | jq -r '.commit')

.ci/bwcVersions

@@ -15,9 +15,9 @@ BWC_VERSION:
   - "8.13.4"
   - "8.14.3"
   - "8.15.5"
-  - "8.16.7"
-  - "8.17.5"
-  - "8.18.0"
+  - "8.16.6"
+  - "8.17.6"
+  - "8.18.1"
   - "8.19.0"
-  - "9.0.0"
+  - "9.0.1"
   - "9.1.0"

.ci/snapshotBwcVersions

@@ -1,7 +1,6 @@
 BWC_VERSION:
-  - "8.16.7"
-  - "8.17.5"
-  - "8.18.0"
+  - "8.17.6"
+  - "8.18.1"
   - "8.19.0"
-  - "9.0.0"
+  - "9.0.1"
   - "9.1.0"

benchmarks/run.sh

@@ -0,0 +1,44 @@
+#!/bin/bash
+#
+# Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+# or more contributor license agreements. Licensed under the "Elastic License
+# 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+# Public License v 1"; you may not use this file except in compliance with, at
+# your election, the "Elastic License 2.0", the "GNU Affero General Public
+# License v3.0 only", or the "Server Side Public License, v 1".
+#
+
+EXTRA=""
+POSITIONAL_ARGS=()
+while [[ $# -gt 0 ]]; do
+  case $1 in
+    --test)
+      # Get inaccurate results quickly by shortening all measurements
+      # to 50ms each and skip self tests.
+      EXTRA="-r 50ms -w 50ms -jvmArgsAppend -DskipSelfTest=true"
+      shift
+      ;;
+    *)
+      POSITIONAL_ARGS+=("$1")
+      shift
+      ;;
+  esac
+done
+
+set -- "${POSITIONAL_ARGS[@]}"
+
+run() {
+  ../gradlew run --args "$2 -rf json $EXTRA"
+  mv jmh-result.json build/benchmarks/$1.json
+}
+
+cd "$(dirname "$0")"
+mkdir -p build/benchmarks
+run 'esql_agg' 'AggregatorBenchmark -pgrouping=none,longs -pfilter=none -pblockType=vector_longs,half_null_longs'
+run 'esql_block_keep_mask' 'BlockKeepMaskBenchmark -pdataTypeAndBlockKind=BytesRef/array,BytesRef/vector,long/array,long/vector'
+run 'esql_block_read' 'BlockReadBenchmark -paccessType=sequential'
+run 'esql_eval' 'EvalBenchmark'
+run 'esql_parse_ip' 'ParseIpBenchmark'
+run 'esql_topn' 'TopNBenchmark'
+run 'esql_values_agg' 'ValuesAggregatorBenchmark'
+run 'esql_values_source_reader' 'ValuesSourceReaderBenchmark'

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/AggregatorBenchmark.java

@@ -113,6 +113,12 @@ public class AggregatorBenchmark {
 
     static {
         // Smoke test all the expected values and force loading subclasses more like prod
+        if (false == "true".equals(System.getProperty("skipSelfTest"))) {
+            selfTest();
+        }
+    }
+
+    static void selfTest() {
         try {
             for (String grouping : AggregatorBenchmark.class.getField("grouping").getAnnotationsByType(Param.class)[0].value()) {
                 for (String op : AggregatorBenchmark.class.getField("op").getAnnotationsByType(Param.class)[0].value()) {