Merge branch 'main' of github.com:elastic/elasticsearch into inference-eis-completion-support

Author: afoucret

benchmarks/build.gradle

@@ -91,6 +91,12 @@ tasks.named("run").configure {
   systemProperty 'es.nativelibs.path', TestUtil.getTestLibraryPath(file("../libs/native/libraries/build/platform/").toString())
 }
 
+tasks.named('test').configure {
+  if (buildParams.getRuntimeJavaVersion().map{ it.majorVersion.toInteger() }.get() >= 21) {
+    jvmArgs '--add-modules=jdk.incubator.vector'
+  }
+}
+
 spotless {
   java {
     // IDEs can sometimes run annotation processors that leave files in

benchmarks/src/main/java/org/elasticsearch/benchmark/_nightly/esql/QueryPlanningBenchmark.java

@@ -52,6 +52,7 @@
 import java.util.LinkedHashMap;
 import java.util.Locale;
 import java.util.Map;
+import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 import static java.util.Collections.emptyMap;
@@ -101,7 +102,7 @@ public void setup() {
             mapping.put("field" + i, new EsField("field-" + i, TEXT, emptyMap(), true, EsField.TimeSeriesFieldType.NONE));
         }
 
-        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD));
+        var esIndex = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD), Set.of());
 
         var functionRegistry = new EsqlFunctionRegistry();
 

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/EvalBenchmark.java

@@ -217,7 +217,12 @@ private static EvalOperator.ExpressionEvaluator evaluator(String operation) {
                 );
                 yield EvalMapper.toEvaluator(
                     FOLD_CONTEXT,
-                    new DateTrunc(Source.EMPTY, new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION), timestamp),
+                    new DateTrunc(
+                        Source.EMPTY,
+                        new Literal(Source.EMPTY, Duration.ofHours(24), DataType.TIME_DURATION),
+                        timestamp,
+                        configuration()
+                    ),
                     layout(timestamp)
                 ).get(driverContext);
             }

benchmarks/src/main/java/org/elasticsearch/benchmark/index/codec/tsdb/TSDBDocValuesMergeBenchmark.java

@@ -27,6 +27,7 @@
 import org.elasticsearch.cluster.metadata.DataStream;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.codec.Elasticsearch92Lucene103Codec;
+import org.elasticsearch.index.codec.tsdb.BinaryDVCompressionMode;
 import org.elasticsearch.index.codec.tsdb.es819.ES819TSDBDocValuesFormat;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -257,7 +258,13 @@ private static IndexWriterConfig createIndexWriterConfig(boolean optimizedMergeE
         );
         config.setLeafSorter(DataStream.TIMESERIES_LEAF_READERS_SORTER);
         config.setMergePolicy(new LogByteSizeMergePolicy());
-        var docValuesFormat = new ES819TSDBDocValuesFormat(4096, 512, optimizedMergeEnabled);
+        var docValuesFormat = new ES819TSDBDocValuesFormat(
+            4096,
+            512,
+            optimizedMergeEnabled,
+            BinaryDVCompressionMode.COMPRESSED_ZSTD_LEVEL_1,
+            true
+        );
         config.setCodec(new Elasticsearch92Lucene103Codec() {
             @Override
             public DocValuesFormat getDocValuesFormatForField(String field) {

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java

@@ -18,7 +18,9 @@
 import org.apache.lucene.util.quantization.OptimizedScalarQuantizer;
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.core.IOUtils;
+import org.elasticsearch.index.codec.vectors.diskbbq.next.ESNextDiskBBQVectorsFormat;
 import org.elasticsearch.simdvec.ES91OSQVectorsScorer;
+import org.elasticsearch.simdvec.ESNextOSQVectorsScorer;
 import org.elasticsearch.simdvec.internal.vectorization.ESVectorizationProvider;
 import org.openjdk.jmh.annotations.Benchmark;
 import org.openjdk.jmh.annotations.BenchmarkMode;
@@ -57,6 +59,9 @@ public class OSQScorerBenchmark {
     @Param({ "384", "782", "1024" })
     int dims;
 
+    @Param({ "1", "2", "4" })
+    int bits;
+
     int length;
 
     int numVectors = ES91OSQVectorsScorer.BULK_SIZE * 10;
@@ -68,8 +73,8 @@ public class OSQScorerBenchmark {
     float centroidDp;
 
     byte[] scratch;
-    ES91OSQVectorsScorer scorerMmap;
-    ES91OSQVectorsScorer scorerNfios;
+    ESNextOSQVectorsScorer scorerMmap;
+    ESNextOSQVectorsScorer scorerNfios;
 
     Directory dirMmap;
     IndexInput inMmap;
@@ -84,7 +89,12 @@ public class OSQScorerBenchmark {
     public void setup() throws IOException {
         Random random = new Random(123);
 
-        this.length = OptimizedScalarQuantizer.discretize(dims, 64) / 8;
+        this.length = switch (bits) {
+            case 1 -> ESNextDiskBBQVectorsFormat.QuantEncoding.ONE_BIT_4BIT_QUERY.getDocPackedLength(dims);
+            case 2 -> ESNextDiskBBQVectorsFormat.QuantEncoding.TWO_BIT_4BIT_QUERY.getDocPackedLength(dims);
+            case 4 -> ESNextDiskBBQVectorsFormat.QuantEncoding.FOUR_BIT_SYMMETRIC.getDocPackedLength(dims);
+            default -> throw new IllegalArgumentException("Unsupported bits: " + bits);
+        };
 
         binaryVectors = new byte[numVectors][length];
         for (byte[] binaryVector : binaryVectors) {
@@ -109,8 +119,14 @@ public void setup() throws IOException {
         outNfios.close();
         inMmap = dirMmap.openInput("vectors", IOContext.DEFAULT);
         inNiofs = dirNiofs.openInput("vectors", IOContext.DEFAULT);
-
-        binaryQueries = new byte[numVectors][4 * length];
+        int binaryQueryLength = switch (bits) {
+            case 1 -> ESNextDiskBBQVectorsFormat.QuantEncoding.ONE_BIT_4BIT_QUERY.getQueryPackedLength(dims);
+            case 2 -> ESNextDiskBBQVectorsFormat.QuantEncoding.TWO_BIT_4BIT_QUERY.getQueryPackedLength(dims);
+            case 4 -> ESNextDiskBBQVectorsFormat.QuantEncoding.FOUR_BIT_SYMMETRIC.getQueryPackedLength(dims);
+            default -> throw new IllegalArgumentException("Unsupported bits: " + bits);
+        };
+
+        binaryQueries = new byte[numVectors][binaryQueryLength];
         for (byte[] binaryVector : binaryVectors) {
             random.nextBytes(binaryVector);
         }
@@ -123,8 +139,26 @@ public void setup() throws IOException {
         centroidDp = random.nextFloat();
 
         scratch = new byte[length];
-        scorerMmap = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inMmap, dims);
-        scorerNfios = ESVectorizationProvider.getInstance().newES91OSQVectorsScorer(inNiofs, dims);
+        final int docBits;
+        final int queryBits = switch (bits) {
+            case 1 -> {
+                docBits = 1;
+                yield 4;
+            }
+            case 2 -> {
+                docBits = 2;
+                yield 4;
+            }
+            case 4 -> {
+                docBits = 4;
+                yield 4;
+            }
+            default -> throw new IllegalArgumentException("Unsupported bits: " + bits);
+        };
+        scorerMmap = ESVectorizationProvider.getInstance()
+            .newESNextOSQVectorsScorer(inMmap, (byte) queryBits, (byte) docBits, dims, length);
+        scorerNfios = ESVectorizationProvider.getInstance()
+            .newESNextOSQVectorsScorer(inNiofs, (byte) queryBits, (byte) docBits, dims, length);
         scratchScores = new float[16];
         corrections = new float[3];
     }
@@ -156,7 +190,7 @@ public void scoreFromMemorySegmentOnlyVectorNiofsVect(Blackhole bh) throws IOExc
         scoreFromMemorySegmentOnlyVector(bh, inNiofs, scorerNfios);
     }
 
-    private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
+    private void scoreFromMemorySegmentOnlyVector(Blackhole bh, IndexInput in, ESNextOSQVectorsScorer scorer) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i++) {
@@ -203,7 +237,7 @@ public void scoreFromMemorySegmentOnlyVectorBulkNiofsVect(Blackhole bh) throws I
         scoreFromMemorySegmentOnlyVectorBulk(bh, inNiofs, scorerNfios);
     }
 
-    private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
+    private void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh, IndexInput in, ESNextOSQVectorsScorer scorer) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i += 16) {
@@ -252,7 +286,7 @@ public void scoreFromMemorySegmentAllBulkNiofsVect(Blackhole bh) throws IOExcept
         scoreFromMemorySegmentAllBulk(bh, inNiofs, scorerNfios);
     }
 
-    private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ES91OSQVectorsScorer scorer) throws IOException {
+    private void scoreFromMemorySegmentAllBulk(Blackhole bh, IndexInput in, ESNextOSQVectorsScorer scorer) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i += 16) {

build-tools-internal/version.properties

@@ -1,5 +1,5 @@
 elasticsearch     = 9.3.0
-lucene            = 10.3.1
+lucene            = 10.3.2
 
 bundled_jdk_vendor = openjdk
 bundled_jdk = 25.0.1+8@2fbf10d8c78e40bd87641c434705079d
@@ -19,6 +19,7 @@ commons_lang3     = 3.9
 google_oauth_client = 1.34.1
 awsv2sdk            = 2.31.78
 reactive_streams    = 1.0.4
+ldapsdk           = 7.0.3
 
 antlr4            = 4.13.1
 # bouncy castle version for non-fips. fips jars use a different version

distribution/docker/src/docker/dockerfiles/cloud_ess_fips/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:dea9b73186c5565a5bfa820a68f63ff497fdc9f06a0afc777159d819c5f01dc4 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a02c67d96cd6ec1b50a055f1f5515e0987b643d001d436088cefc02e5b786bf9 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -104,7 +104,7 @@ WORKDIR /usr/share/elasticsearch/config
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:dea9b73186c5565a5bfa820a68f63ff497fdc9f06a0afc777159d819c5f01dc4
+FROM docker.elastic.co/wolfi/chainguard-base-fips:latest@sha256:a02c67d96cd6ec1b50a055f1f5515e0987b643d001d436088cefc02e5b786bf9
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

distribution/docker/src/docker/dockerfiles/wolfi/Dockerfile

@@ -25,7 +25,7 @@
 # Extract Elasticsearch artifact
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:401d8684fd5932c178d7cf524cb05a5088d0e4f74e886e2d0a3b4b5127349a49 AS builder
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:25397820ef168d951339e69cedb5460f354cea842e2af367e33b89e988c51615 AS builder
 
 # Install required packages to extract the Elasticsearch distribution
 RUN <%= retry.loop(package_manager, "export DEBIAN_FRONTEND=noninteractive && ${package_manager} update && ${package_manager} update && ${package_manager} add --no-cache curl") %>
@@ -80,7 +80,7 @@ RUN sed -i -e 's/ES_DISTRIBUTION_TYPE=tar/ES_DISTRIBUTION_TYPE=docker/' bin/elas
 # Add entrypoint
 ################################################################################
 
-FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:401d8684fd5932c178d7cf524cb05a5088d0e4f74e886e2d0a3b4b5127349a49
+FROM docker.elastic.co/wolfi/chainguard-base:latest@sha256:25397820ef168d951339e69cedb5460f354cea842e2af367e33b89e988c51615
 
 RUN <%= retry.loop(package_manager,
           "export DEBIAN_FRONTEND=noninteractive && \n" +

docs/Versions.asciidoc

@@ -1,8 +1,8 @@
 
 include::{docs-root}/shared/versions/stack/{source_branch}.asciidoc[]
 
-:lucene_version:        10.3.1
-:lucene_version_path:   10_3_1
+:lucene_version:        10.3.2
+:lucene_version_path:   10_3_2
 :jdk:                   11.0.2
 :jdk_major:             11
 :build_type:            tar

docs/changelog/128602.yaml

@@ -0,0 +1,6 @@
+pr: 128602
+summary: ChangePointDetector now correctly detects when a constant function returns a different value
+area: Machine Learning
+type: bug
+issues:
+ - 127517