Merge branch '8.19' of https://github.com/elastic/elasticsearch into esql-inference-feature-manual-backport

Author: afoucret

benchmarks/src/main/java/org/elasticsearch/benchmark/compute/operator/ValuesAggregatorBenchmark.java

@@ -21,10 +21,13 @@
 import org.elasticsearch.compute.data.Block;
 import org.elasticsearch.compute.data.BlockFactory;
 import org.elasticsearch.compute.data.BytesRefBlock;
+import org.elasticsearch.compute.data.BytesRefVector;
 import org.elasticsearch.compute.data.ElementType;
 import org.elasticsearch.compute.data.IntBlock;
+import org.elasticsearch.compute.data.IntVector;
 import org.elasticsearch.compute.data.LongBlock;
 import org.elasticsearch.compute.data.LongVector;
+import org.elasticsearch.compute.data.OrdinalBytesRefVector;
 import org.elasticsearch.compute.data.Page;
 import org.elasticsearch.compute.operator.AggregationOperator;
 import org.elasticsearch.compute.operator.DriverContext;
@@ -275,11 +278,18 @@ private static Block dataBlock(int groups, String dataType) {
         int blockLength = blockLength(groups);
         return switch (dataType) {
             case BYTES_REF -> {
-                try (BytesRefBlock.Builder builder = blockFactory.newBytesRefBlockBuilder(blockLength)) {
+                try (
+                    BytesRefVector.Builder dict = blockFactory.newBytesRefVectorBuilder(blockLength);
+                    IntVector.Builder ords = blockFactory.newIntVectorBuilder(blockLength)
+                ) {
+                    final int dictLength = Math.min(blockLength, KEYWORDS.length);
+                    for (int i = 0; i < dictLength; i++) {
+                        dict.appendBytesRef(KEYWORDS[i]);
+                    }
                     for (int i = 0; i < blockLength; i++) {
-                        builder.appendBytesRef(KEYWORDS[i % KEYWORDS.length]);
+                        ords.appendInt(i % dictLength);
                     }
-                    yield builder.build();
+                    yield new OrdinalBytesRefVector(ords.build(), dict.build()).asBlock();
                 }
             }
             case INT -> {

benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/OptimizedTextBenchmark.java

@@ -0,0 +1,108 @@
+/*
+ * Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
+ * or more contributor license agreements. Licensed under the "Elastic License
+ * 2.0", the "GNU Affero General Public License v3.0 only", and the "Server Side
+ * Public License v 1"; you may not use this file except in compliance with, at
+ * your election, the "Elastic License 2.0", the "GNU Affero General Public
+ * License v3.0 only", or the "Server Side Public License, v 1".
+ */
+
+package org.elasticsearch.benchmark.xcontent;
+
+import org.elasticsearch.benchmark.index.mapper.MapperServiceFactory;
+import org.elasticsearch.common.UUIDs;
+import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.common.logging.LogConfigurator;
+import org.elasticsearch.index.mapper.MapperService;
+import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.xcontent.XContentBuilder;
+import org.elasticsearch.xcontent.XContentFactory;
+import org.elasticsearch.xcontent.XContentType;
+import org.openjdk.jmh.annotations.Benchmark;
+import org.openjdk.jmh.annotations.BenchmarkMode;
+import org.openjdk.jmh.annotations.Fork;
+import org.openjdk.jmh.annotations.Level;
+import org.openjdk.jmh.annotations.Measurement;
+import org.openjdk.jmh.annotations.Mode;
+import org.openjdk.jmh.annotations.OutputTimeUnit;
+import org.openjdk.jmh.annotations.Param;
+import org.openjdk.jmh.annotations.Scope;
+import org.openjdk.jmh.annotations.Setup;
+import org.openjdk.jmh.annotations.State;
+import org.openjdk.jmh.annotations.Threads;
+import org.openjdk.jmh.annotations.Warmup;
+import org.openjdk.jmh.infra.Blackhole;
+
+import java.io.IOException;
+import java.util.Random;
+import java.util.concurrent.TimeUnit;
+
+/**
+ * Benchmark to measure indexing performance of keyword fields. Used to measure performance impact of skipping
+ * UTF-8 to UTF-16 conversion during document parsing.
+ */
+@BenchmarkMode(Mode.AverageTime)
+@OutputTimeUnit(TimeUnit.MILLISECONDS)
+@State(Scope.Benchmark)
+@Fork(1)
+@Threads(1)
+@Warmup(iterations = 1)
+@Measurement(iterations = 5)
+public class OptimizedTextBenchmark {
+    static {
+        // For Elasticsearch900Lucene101Codec:
+        LogConfigurator.loadLog4jPlugins();
+        LogConfigurator.configureESLogging();
+        LogConfigurator.setNodeName("test");
+    }
+
+    /**
+     * Total number of documents to index.
+     */
+    @Param("1048576")
+    private int nDocs;
+
+    private MapperService mapperService;
+    private SourceToParse[] sources;
+
+    private String randomValue(int length) {
+        final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+        Random random = new Random();
+        StringBuilder builder = new StringBuilder(length);
+        for (int i = 0; i < length; i++) {
+            builder.append(CHARS.charAt(random.nextInt(CHARS.length())));
+        }
+        return builder.toString();
+    }
+
+    @Setup(Level.Trial)
+    public void setup() throws IOException {
+        mapperService = MapperServiceFactory.create("""
+            {
+                "_doc": {
+                    "dynamic": false,
+                    "properties": {
+                        "field": {
+                            "type": "keyword"
+                        }
+                    }
+                }
+            }
+            """);
+
+        sources = new SourceToParse[nDocs];
+        for (int i = 0; i < nDocs; i++) {
+            XContentBuilder b = XContentFactory.jsonBuilder();
+            b.startObject().field("field", randomValue(8)).endObject();
+            sources[i] = new SourceToParse(UUIDs.randomBase64UUID(), BytesReference.bytes(b), XContentType.JSON);
+        }
+    }
+
+    @Benchmark
+    public void indexDocuments(final Blackhole bh) {
+        final var mapper = mapperService.documentMapper();
+        for (int i = 0; i < nDocs; i++) {
+            bh.consume(mapper.parse(sources[i]));
+        }
+    }
+}

distribution/docker/src/docker/dockerfiles/default/Dockerfile

@@ -29,18 +29,15 @@ RUN apt-get update -y && DEBIAN_FRONTEND=noninteractive apt-get install -y curl
 # The tini GitHub page gives instructions for verifying the binary using
 # gpg, but the keyservers are slow to return the key and this can fail the
 # build. Instead, we check the binary against the published checksum.
-RUN set -eux ; \\
-    tini_bin="" ; \\
+RUN set -eux; \\
     case "\$(arch)" in \\
-        aarch64) tini_bin='tini-arm64' ;; \\
-        x86_64)  tini_bin='tini-amd64' ;; \\
-        *) echo >&2 ; echo >&2 "Unsupported architecture \$(arch)" ; echo >&2 ; exit 1 ;; \\
+        aarch64) tini_bin='tini-arm64'; tini_sum='07952557df20bfd2a95f9bef198b445e006171969499a1d361bd9e6f8e5e0e81' ;; \\
+        x86_64)  tini_bin='tini-amd64'; tini_sum='93dcc18adc78c65a028a84799ecf8ad40c936fdfc5f2a57b1acda5a8117fa82c' ;; \\
+        *) echo >&2 "Unsupported architecture \$arch"; exit 1 ;; \\
     esac ; \\
-    curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin} ; \\
-    curl --retry 10 -S -L -O https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}.sha256sum ; \\
-    sha256sum -c \${tini_bin}.sha256sum ; \\
-    rm \${tini_bin}.sha256sum ; \\
-    mv \${tini_bin} /bin/tini ; \\
+    curl -f --retry 10 -S -L -o /tmp/tini https://github.com/krallin/tini/releases/download/v0.19.0/\${tini_bin}; \\
+    echo "\${tini_sum}  /tmp/tini" | sha256sum -c -; \\
+    mv /tmp/tini /bin/tini; \\
     chmod 0555 /bin/tini
 
 RUN mkdir /usr/share/elasticsearch

docs/changelog/127797.yaml

@@ -0,0 +1,6 @@
+pr: 127797
+summary: "Date nanos implicit casting in union types option #2"
+area: ES|QL
+type: enhancement
+issues:
+ - 110009

docs/changelog/127849.yaml

@@ -0,0 +1,5 @@
+pr: 127849
+summary: Optimize ordinal inputs in Values aggregation
+area: "ES|QL"
+type: enhancement
+issues: []

docs/changelog/128890.yaml

@@ -0,0 +1,5 @@
+pr: 128890
+summary: Improve cache invalidation in IdP SP cache
+area: IdentityProvider
+type: bug
+issues: []

docs/reference/esql/functions/description/match.asciidoc

@@ -18,6 +18,7 @@ If you don’t specify an inference endpoint, the `inference_id` field defaults
 
 Using `semantic_text`, you won't need to specify how to generate embeddings for your data, or how to index it.
 The {infer} endpoint automatically determines the embedding generation, indexing, and query to use.
+Newly created indices with `semantic_text` fields using dense embeddings will be <<dense-vector-quantization,quantized>> to `bbq_hnsw` automatically.
 
 If you use the preconfigured `.elser-2-elasticsearch` endpoint, you can set up `semantic_text` with the following API request:
 
@@ -225,7 +226,8 @@ In these cases - when you use `sparse_vector` or `dense_vector` field types inst
 For indices containing `semantic_text` fields, updates that use scripts have the following behavior:
 
 * Are supported through the https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-update[Update API].
-* Are not supported through the https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk-1[Bulk API] and will fail. Even if the script targets non-`semantic_text` fields, the update will fail when the index contains a `semantic_text` field.
+* Are not supported through the https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-bulk-1[Bulk API] and will fail.
+Even if the script targets non-`semantic_text` fields, the update will fail when the index contains a `semantic_text` field.
 
 [discrete]
 [[copy-to-support]]