Merge branch 'main' into dbbq_bulk_size

Author: tteofili

.buildkite/pipelines/periodic-java-ea.yml

@@ -385,8 +385,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.7 / bwc
-        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.0.7#bwcTest
+      - label: 9.0.8 / bwc
+        command: .ci/scripts/run-gradle.sh -Druntime.java=$$JAVA_EA_VERSION -Dbwc.checkout.align=true v9.0.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -395,7 +395,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.7
+          BWC_VERSION: 9.0.8
         retry:
           automatic:
             - exit_status: "-1"

.buildkite/pipelines/periodic-packaging.yml

@@ -352,8 +352,8 @@ steps:
         env:
           BWC_VERSION: 8.19.8
 
-      - label: "{{matrix.image}} / 9.0.7 / packaging-tests-upgrade"
-        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.7
+      - label: "{{matrix.image}} / 9.0.8 / packaging-tests-upgrade"
+        command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.0.8
         timeout_in_minutes: 300
         matrix:
           setup:
@@ -366,7 +366,7 @@ steps:
           machineType: custom-16-32768
           buildDirectory: /dev/shm/bk
         env:
-          BWC_VERSION: 9.0.7
+          BWC_VERSION: 9.0.8
 
       - label: "{{matrix.image}} / 9.1.8 / packaging-tests-upgrade"
         command: ./.ci/scripts/packaging-test.sh -Dbwc.checkout.align=true destructiveDistroUpgradeTest.v9.1.8

.buildkite/pipelines/periodic.yml

@@ -382,8 +382,8 @@ steps:
             - signal_reason: agent_stop
               limit: 3
 
-      - label: 9.0.7 / bwc
-        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.0.7#bwcTest
+      - label: 9.0.8 / bwc
+        command: .ci/scripts/run-gradle.sh --continue -Dbwc.checkout.align=true v9.0.8#bwcTest
         timeout_in_minutes: 300
         agents:
           provider: gcp
@@ -392,7 +392,7 @@ steps:
           buildDirectory: /dev/shm/bk
           preemptible: true
         env:
-          BWC_VERSION: 9.0.7
+          BWC_VERSION: 9.0.8
         retry:
           automatic:
             - exit_status: "-1"

.ci/bwcVersions

@@ -19,7 +19,7 @@ BWC_VERSION:
   - "8.17.10"
   - "8.18.8"
   - "8.19.8"
-  - "9.0.7"
+  - "9.0.8"
   - "9.1.8"
   - "9.2.2"
   - "9.3.0"

.gitattributes

@@ -4,12 +4,13 @@ CHANGELOG.asciidoc  merge=union
 # Windows
 build-tools-internal/src/test/resources/org/elasticsearch/gradle/internal/release/*.asciidoc text eol=lf
 
+# ESQL parsing and source generated related assets
 x-pack/plugin/esql/compute/src/main/generated/** linguist-generated=true
 x-pack/plugin/esql/compute/src/main/generated-src/** linguist-generated=true
 x-pack/plugin/esql/src/main/antlr/*.tokens linguist-generated=true
 x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*.interp linguist-generated=true
-x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseLexer*.java linguist-generated=true
-x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/EsqlBaseParser*.java linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*BaseLexer*.java linguist-generated=true
+x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/parser/*BaseParser*.java linguist-generated=true
 x-pack/plugin/esql/src/main/generated/** linguist-generated=true
 x-pack/plugin/esql/src/main/generated-src/** linguist-generated=true
 

benchmarks/build.gradle

@@ -91,6 +91,12 @@ tasks.named("run").configure {
   systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())
 }
 
+tasks.named('test').configure {
+  if (buildParams.getRuntimeJavaVersion().map{ it.majorVersion.toInteger() }.get() >= 21) {
+    jvmArgs '--add-modules=jdk.incubator.vector'
+  }
+}
+
 spotless {
   java {
     // IDEs can sometimes run annotation processors that leave files in

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -52,6 +52,7 @@
 import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyMap;
@@ -101,7 +102,7 @@ public void setup() {
             mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true, EsField.TimeSeriesFieldType.NONE));
         }
 
-        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
+        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Map.of(), Map.of(), Set.of());
 
         var functionRegistry = new EsqlFunctionRegistry();
 

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.codec.Elasticsearch92Lucene103Codec;
+import org.elasticsearch.index.codec.tsdb.BinaryDVCompressionMode;
 import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -257,7 +258,13 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
         );
         config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
         config.setMergePolicy(new LogByteSizeMergePolicy());
-        var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
+        var docValuesFormat = new ES819TSDBDocValuesFormat(
+            4096,
+            512,
+            optimizedMergeEnabled,
+            BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1,
+            true
+        );
         config.setCodec(new Elasticsearch92Lucene103Codec() {
             @Override
             public DocValuesFormat getDocValuesFormatForField(String field) {

…k/vector/ComputeNeighboursBenchmark.java …tization/ComputeNeighboursBenchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/vector/ComputeNeighboursBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/ComputeNeighboursBenchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/vector/ComputeNeighboursBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/ComputeNeighboursBenchmark.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.vector;
+package org.elasticsearch.benchmark.vector.quantization;
 
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.codec.vectors.cluster.NeighborHood;

…r/OptimizedScalarQuantizerBenchmark.java …n/OptimizedScalarQuantizerBenchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/OptimizedScalarQuantizerBenchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/OptimizedScalarQuantizerBenchmark.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.vector;
+package org.elasticsearch.benchmark.vector.quantization;
 
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.elasticsearch.common.logging.LogConfigurator;