Merge remote-tracking branch 'origin/main' into ankikuma-09162025/ReshardSplitRequestOnSourceTwoPass

Author: Tim-Brooks

.buildkite/pipelines/intake.yml

@@ -65,7 +65,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.8", "8.19.5", "9.0.8", "9.1.6", "9.2.0", "9.3.0"]
+            BWC_VERSION: ["8.18.9", "8.19.6", "9.0.9", "9.1.6", "9.2.0", "9.3.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-java-ea.yml

@@ -347,8 +347,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.8 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.18.8#bwcTest
+      - label: 8.18.9 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.18.9#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -357,7 +357,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.8
+          BWC_VERSION: 8.18.9
         retry:
           automatic:
             - exit_status: "-1"
@@ -366,8 +366,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.19.5#bwcTest
+      - label: 8.19.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v8.19.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -376,7 +376,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.5
+          BWC_VERSION: 8.19.6
         retry:
           automatic:
             - exit_status: "-1"
@@ -385,8 +385,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.8 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.0.8#bwcTest
+      - label: 9.0.9 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.0.9#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -395,7 +395,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.8
+          BWC_VERSION: 9.0.9
         retry:
           automatic:
             - exit_status: "-1"
@@ -538,7 +538,7 @@ steps:
         timeout_in_minutes: 300
         matrix:
           setup:
-            BWC_VERSION: ["8.18.8", "8.19.5", "9.0.8", "9.1.6", "9.2.0", "9.3.0"]
+            BWC_VERSION: ["8.18.9", "8.19.6", "9.0.9", "9.1.6", "9.2.0", "9.3.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/pipelines/periodic-packaging.yml

@@ -318,8 +318,8 @@ steps:
         env:
           BWC_VERSION: 8.17.10
 
-      - label: "{{matrix.image}} / 8.18.8 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.8
+      - label: "{{matrix.image}} / 8.18.9 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.18.9
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -332,10 +332,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.18.8
+          BWC_VERSION: 8.18.9
 
-      - label: "{{matrix.image}} / 8.19.5 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.5
+      - label: "{{matrix.image}} / 8.19.6 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v8.19.6
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -348,10 +348,10 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 8.19.5
+          BWC_VERSION: 8.19.6
 
-      - label: "{{matrix.image}} / 9.0.8 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.8
+      - label: "{{matrix.image}} / 9.0.9 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.9
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -364,7 +364,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.8
+          BWC_VERSION: 9.0.9
 
       - label: "{{matrix.image}} / 9.1.6 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.6

.buildkite/pipelines/periodic.yml

@@ -344,8 +344,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.18.8 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.8#bwcTest
+      - label: 8.18.9 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.18.9#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -354,7 +354,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.18.8
+          BWC_VERSION: 8.18.9
         retry:
           automatic:
             - exit_status: "-1"
@@ -363,8 +363,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 8.19.5 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.5#bwcTest
+      - label: 8.19.6 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v8.19.6#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -373,7 +373,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 8.19.5
+          BWC_VERSION: 8.19.6
         retry:
           automatic:
             - exit_status: "-1"
@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.8 / bwc
-        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.8#bwcTest
+      - label: 9.0.9 / bwc
+        command: .ci/scripts/run-gradle.sh -Dbwc.checkout.align=true v9.0.9#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.8
+          BWC_VERSION: 9.0.9
         retry:
           automatic:
             - exit_status: "-1"
@@ -525,7 +525,7 @@ steps:
           setup:
             ES_RUNTIME_JAVA:
               - openjdk21
-            BWC_VERSION: ["8.18.8", "8.19.5", "9.0.8", "9.1.6", "9.2.0", "9.3.0"]
+            BWC_VERSION: ["8.18.9", "8.19.6", "9.0.9", "9.1.6", "9.2.0", "9.3.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404
@@ -569,7 +569,7 @@ steps:
             ES_RUNTIME_JAVA:
               - openjdk21
               - openjdk23
-            BWC_VERSION: ["8.18.8", "8.19.5", "9.0.8", "9.1.6", "9.2.0", "9.3.0"]
+            BWC_VERSION: ["8.18.9", "8.19.6", "9.0.9", "9.1.6", "9.2.0", "9.3.0"]
         agents:
           provider: gcp
           image: family/elasticsearch-ubuntu-2404

.buildkite/scripts/generate-pr-performance-benchmark.sh

@@ -46,6 +46,7 @@ steps:
         CONFIGURATION_NAME: ${GITHUB_PR_COMMENT_VAR_BENCHMARK}
         ENV_ID: ${env_id_baseline}
         REVISION: ${merge_base}
+        BENCHMARK_TYPE: baseline
   - label: Trigger contender benchmark with ${GITHUB_PR_TRIGGERED_SHA:0:7}
     trigger: elasticsearch-performance-esbench-pr
     build:
@@ -56,6 +57,7 @@ steps:
         ENV_ID: ${env_id_contender}
         ES_REPO_URL: https://github.com/${GITHUB_PR_OWNER}/${GITHUB_PR_REPO}.git
         REVISION: ${GITHUB_PR_TRIGGERED_SHA}
+        BENCHMARK_TYPE: contender
   - wait: ~
   - label: Update PR comment and Buildkite annotation
     command: |

.ci/bwcVersions

@@ -17,9 +17,9 @@ BWC_VERSION:
   - "8.15.5"
   - "8.16.6"
   - "8.17.10"
-  - "8.18.8"
-  - "8.19.5"
-  - "9.0.8"
+  - "8.18.9"
+  - "8.19.6"
+  - "9.0.9"
   - "9.1.6"
   - "9.2.0"
   - "9.3.0"

.ci/snapshotBwcVersions

@@ -1,7 +1,7 @@
 BWC_VERSION:
-  - "8.18.8"
-  - "8.19.5"
-  - "9.0.8"
+  - "8.18.9"
+  - "8.19.6"
+  - "9.0.9"
   - "9.1.6"
   - "9.2.0"
   - "9.3.0"

.gitignore

@@ -46,6 +46,7 @@ build/
 **/.local*
 .vagrant/
 /logs/
+**/target/
 
 # osx stuff
 .DS_Store

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/ValuesSourceReaderBenchmark.java

@@ -51,6 +51,7 @@
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.mapper.BlockLoader;
 import org.elasticsearch.index.mapper.FieldNamesFieldMapper;
+import org.elasticsearch.index.mapper.IndexType;
 import org.elasticsearch.index.mapper.KeywordFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
 import org.elasticsearch.index.mapper.NumberFieldMapper;
@@ -290,9 +291,8 @@ private static BlockLoader numericBlockLoader(WhereAndBaseName w, NumberFieldMap
         return new NumberFieldMapper.NumberFieldType(
             w.name,
             numberType,
-            true,
+            IndexType.points(true, docValues),
             stored,
-            docValues,
             true,
             null,
             Map.of(),

benchmarks/src/main/java/org/elasticsearch/benchmark/bytes/RecyclerBytesStreamOutputBenchmark.java

@@ -10,6 +10,7 @@
 package org.elasticsearch.benchmark.bytes;
 
 import org.apache.lucene.util.BytesRef;
+import org.elasticsearch.benchmark.common.util.UTF8StringBytesBenchmark;
 import org.elasticsearch.common.io.stream.RecyclerBytesStreamOutput;
 import org.elasticsearch.common.recycler.Recycler;
 import org.openjdk.jmh.annotations.Benchmark;
@@ -65,10 +66,10 @@ public void initResults() throws IOException {
         // We use weights to generate certain sized UTF-8 characters and vInts. However, there is still some non-determinism which could
         // impact direct comparisons run-to-run
 
-        shortString = generateAsciiString(20);
-        longString = generateAsciiString(100);
-        nonAsciiString = generateUtf8String(200);
-        veryLongString = generateAsciiString(800);
+        shortString = UTF8StringBytesBenchmark.generateAsciiString(20);
+        longString = UTF8StringBytesBenchmark.generateAsciiString(100);
+        nonAsciiString = UTF8StringBytesBenchmark.generateUTF8String(200);
+        veryLongString = UTF8StringBytesBenchmark.generateAsciiString(800);
         // vint values for benchmarking
         vints = new int[1000];
         for (int i = 0; i < vints.length; i++) {
@@ -143,49 +144,6 @@ public void writeVInt() throws IOException {
         }
     }
 
-    public static String generateAsciiString(int n) {
-        ThreadLocalRandom random = ThreadLocalRandom.current();
-        StringBuilder sb = new StringBuilder(n);
-
-        for (int i = 0; i < n; i++) {
-            int ascii = random.nextInt(128);
-            sb.append((char) ascii);
-        }
-
-        return sb.toString();
-    }
-
-    public static String generateUtf8String(int n) {
-        ThreadLocalRandom random = ThreadLocalRandom.current();
-        StringBuilder sb = new StringBuilder(n);
-
-        for (int i = 0; i < n; i++) {
-            int codePoint;
-            int probability = random.nextInt(100);
-
-            if (probability < 85) {
-                // 1-byte UTF-8 (ASCII range)
-                // 0x0000 to 0x007F
-                codePoint = random.nextInt(0x0080);
-            } else if (probability < 95) {
-                // 2-byte UTF-8
-                // 0x0080 to 0x07FF
-                codePoint = random.nextInt(0x0080, 0x0800);
-            } else {
-                // 3-byte UTF-8
-                // 0x0800 to 0xFFFF
-                do {
-                    codePoint = random.nextInt(0x0800, 0x10000);
-                    // Skip surrogate pairs (0xD800-0xDFFF)
-                } while (codePoint >= 0xD800 && codePoint <= 0xDFFF);
-            }
-
-            sb.appendCodePoint(codePoint);
-        }
-
-        return sb.toString();
-    }
-
     private record BenchmarkRecycler(AtomicReference<BytesRef> bytesRef) implements Recycler<BytesRef> {
 
         @Override