Merge branch 'main' of github.com:elastic/elasticsearch into reshard-install-source-search-filter

Author: bcully

benchmarks/build.gradle

@@ -91,6 +91,12 @@ tasks.named("run").configure {
   systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())
 }
 
+tasks.named('test').configure {
+  if (buildParams.getRuntimeJavaVersion().map{ it.majorVersion.toInteger() }.get() >= 21) {
+    jvmArgs '--add-modules=jdk.incubator.vector'
+  }
+}
+
 spotless {
   java {
     // IDEs can sometimes run annotation processors that leave files in

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -52,6 +52,7 @@
 import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyMap;
@@ -101,7 +102,7 @@ public void setup() {
             mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true, EsField.TimeSeriesFieldType.NONE));
         }
 
-        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
+        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Set.of());
 
         var functionRegistry = new EsqlFunctionRegistry();
 

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.codec.Elasticsearch92Lucene103Codec;
+import org.elasticsearch.index.codec.tsdb.BinaryDVCompressionMode;
 import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -257,7 +258,13 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
         );
         config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
         config.setMergePolicy(new LogByteSizeMergePolicy());
-        var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
+        var docValuesFormat = new ES819TSDBDocValuesFormat(
+            4096,
+            512,
+            optimizedMergeEnabled,
+            BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1,
+            true
+        );
         config.setCodec(new Elasticsearch92Lucene103Codec() {
             @Override
             public DocValuesFormat getDocValuesFormatForField(String field) {

…k/vector/ComputeNeighboursBenchmark.java …tization/ComputeNeighboursBenchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/vector/ComputeNeighboursBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/ComputeNeighboursBenchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/vector/ComputeNeighboursBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/ComputeNeighboursBenchmark.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.vector;
+package org.elasticsearch.benchmark.vector.quantization;
 
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.codec.vectors.cluster.NeighborHood;

…r/OptimizedScalarQuantizerBenchmark.java …n/OptimizedScalarQuantizerBenchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/OptimizedScalarQuantizerBenchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OptimizedScalarQuantizerBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/OptimizedScalarQuantizerBenchmark.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.vector;
+package org.elasticsearch.benchmark.vector.quantization;
 
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.elasticsearch.common.logging.LogConfigurator;

…chmark/vector/PackAsBinaryBenchmark.java …/quantization/PackAsBinaryBenchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/vector/PackAsBinaryBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/PackAsBinaryBenchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/vector/PackAsBinaryBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/PackAsBinaryBenchmark.java

@@ -6,7 +6,7 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-package org.elasticsearch.benchmark.vector;
+package org.elasticsearch.benchmark.vector.quantization;
 
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.codec.vectors.BQVectorUtils;

…k/vector/TransposeHalfByteBenchmark.java …tization/TransposeHalfByteBenchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/vector/TransposeHalfByteBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/TransposeHalfByteBenchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/vector/TransposeHalfByteBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/quantization/TransposeHalfByteBenchmark.java

@@ -6,7 +6,7 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-package org.elasticsearch.benchmark.vector;
+package org.elasticsearch.benchmark.vector.quantization;
 
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.codec.vectors.BQVectorUtils;

…chmark/vector/DistanceBulkBenchmark.java …r/VectorScorerDistanceBulkBenchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceBulkBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerDistanceBulkBenchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceBulkBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerDistanceBulkBenchmark.java

@@ -6,7 +6,7 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-package org.elasticsearch.benchmark.vector;
+package org.elasticsearch.benchmark.vector.scorer;
 
 import org.apache.lucene.util.VectorUtil;
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
@@ -38,7 +38,7 @@
 @Measurement(iterations = 5, time = 1)
 // engage some noise reduction
 @Fork(value = 1)
-public class DistanceBulkBenchmark {
+public class VectorScorerDistanceBulkBenchmark {
 
     static {
         LogConfigurator.configureESLogging(); // native access requires logging to be initialized

…rk/vector/DistanceFunctionBenchmark.java …ctorScorerDistanceFunctionBenchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerDistanceFunctionBenchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/vector/DistanceFunctionBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerDistanceFunctionBenchmark.java

@@ -7,7 +7,7 @@
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
 
-package org.elasticsearch.benchmark.vector;
+package org.elasticsearch.benchmark.vector.scorer;
 
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.logging.LogConfigurator;
@@ -49,9 +49,9 @@
 @Measurement(iterations = 2)
 @BenchmarkMode(Mode.AverageTime)
 @OutputTimeUnit(TimeUnit.NANOSECONDS)
-@OperationsPerInvocation(DistanceFunctionBenchmark.OPERATIONS)
+@OperationsPerInvocation(VectorScorerDistanceFunctionBenchmark.OPERATIONS)
 @State(Scope.Benchmark)
-public class DistanceFunctionBenchmark {
+public class VectorScorerDistanceFunctionBenchmark {
 
     public static final int OPERATIONS = 25000;
 
@@ -223,7 +223,7 @@ public void findBenchmarkImpl() {
     }
 
     @Fork(1)
-    @Benchmark
+    // @Benchmark
     public void benchmark(Blackhole blackhole) {
         for (int i = 0; i < OPERATIONS; ++i) {
             blackhole.consume(benchmarkImpl.getAsDouble());

…enchmark/vector/Int4ScorerBenchmark.java …or/scorer/VectorScorerInt4Benchmark.javabenchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int4ScorerBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt4Benchmark.java benchmarks/src/main/java/org/elasticsearch/benchmark/vector/Int4ScorerBenchmark.java renamed to benchmarks/src/main/java/org/elasticsearch/benchmark/vector/scorer/VectorScorerInt4Benchmark.java

@@ -6,7 +6,7 @@
  * your election, the "Elastic License 2.0", the "GNU Affero General Public
  * License v3.0 only", or the "Server Side Public License, v 1".
  */
-package org.elasticsearch.benchmark.vector;
+package org.elasticsearch.benchmark.vector.scorer;
 
 import org.apache.lucene.index.VectorSimilarityFunction;
 import org.apache.lucene.store.Directory;
@@ -48,7 +48,7 @@
 @Measurement(iterations = 5, time = 1)
 // engage some noise reduction
 @Fork(value = 1)
-public class Int4ScorerBenchmark {
+public class VectorScorerInt4Benchmark {
 
     static {
         LogConfigurator.configureESLogging(); // native access requires logging to be initialized