Merge branch 'main' into likeListPushdown_v3

Author: julian-elastic

benchmarks/build.gradle

@@ -41,6 +41,7 @@ dependencies {
   }
   api(project(':libs:h3'))
   api(project(':modules:aggregations'))
+  implementation project(':modules:mapper-extras');
   api(project(':x-pack:plugin:esql-core'))
   api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -38,11 +39,16 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 public class MapperServiceFactory {
 
     public static MapperService create(String mappings) {
+        return create(mappings, Collections.emptyList());
+    }
+
+    public static MapperService create(String mappings, List<MapperPlugin> mapperPlugins) {
         Settings settings = Settings.builder()
             .put("index.number_of_replicas", 0)
             .put("index.number_of_shards", 1)
@@ -51,7 +57,7 @@ public static MapperService create(String mappings) {
             .build();
         IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();
         IndexSettings indexSettings = new IndexSettings(meta, settings);
-        MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
+        MapperRegistry mapperRegistry = new IndicesModule(mapperPlugins).getMapperRegistry();
 
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
         BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);

benchmarks/src/main/java/org/elasticsearch/benchmark/vector/OSQScorerBenchmark.java

@@ -126,7 +126,10 @@ public void scoreFromArray(Blackhole bh) throws IOException {
                 in.readFloats(corrections, 0, corrections.length);
                 int addition = Short.toUnsignedInt(in.readShort());
                 float score = scorer.score(
-                    result,
+                    result.lowerInterval(),
+                    result.upperInterval(),
+                    result.quantizedComponentSum(),
+                    result.additionalCorrection(),
                     VectorSimilarityFunction.EUCLIDEAN,
                     centroidDp,
                     corrections[0],
@@ -150,7 +153,10 @@ public void scoreFromMemorySegmentOnlyVector(Blackhole bh) throws IOException {
                 in.readFloats(corrections, 0, corrections.length);
                 int addition = Short.toUnsignedInt(in.readShort());
                 float score = scorer.score(
-                    result,
+                    result.lowerInterval(),
+                    result.upperInterval(),
+                    result.quantizedComponentSum(),
+                    result.additionalCorrection(),
                     VectorSimilarityFunction.EUCLIDEAN,
                     centroidDp,
                     corrections[0],
@@ -175,7 +181,10 @@ public void scoreFromMemorySegmentOnlyVectorBulk(Blackhole bh) throws IOExceptio
                     in.readFloats(corrections, 0, corrections.length);
                     int addition = Short.toUnsignedInt(in.readShort());
                     float score = scorer.score(
-                        result,
+                        result.lowerInterval(),
+                        result.upperInterval(),
+                        result.quantizedComponentSum(),
+                        result.additionalCorrection(),
                         VectorSimilarityFunction.EUCLIDEAN,
                         centroidDp,
                         corrections[0],
@@ -196,7 +205,16 @@ public void scoreFromMemorySegmentAllBulk(Blackhole bh) throws IOException {
         for (int j = 0; j < numQueries; j++) {
             in.seek(0);
             for (int i = 0; i < numVectors; i += 16) {
-                scorer.scoreBulk(binaryQueries[j], result, VectorSimilarityFunction.EUCLIDEAN, centroidDp, scratchScores);
+                scorer.scoreBulk(
+                    binaryQueries[j],
+                    result.lowerInterval(),
+                    result.upperInterval(),
+                    result.quantizedComponentSum(),
+                    result.additionalCorrection(),
+                    VectorSimilarityFunction.EUCLIDEAN,
+                    centroidDp,
+                    scratchScores
+                );
                 bh.consume(scratchScores);
             }
         }

benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/OptimizedTextBenchmark.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
@@ -34,6 +35,7 @@
 import org.openjdk.jmh.infra.Blackhole;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
@@ -66,7 +68,7 @@ public class OptimizedTextBenchmark {
     private SourceToParse[] sources;
 
     private String randomValue(int length) {
-        final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+        final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ";
         Random random = new Random();
         StringBuilder builder = new StringBuilder(length);
         for (int i = 0; i < length; i++) {
@@ -83,17 +85,17 @@ public void setup() throws IOException {
                     "dynamic": false,
                     "properties": {
                         "field": {
-                            "type": "keyword"
+                            "type": "match_only_text"
                         }
                     }
                 }
             }
-            """);
+            """, List.of(new MapperExtrasPlugin()));
 
         sources = new SourceToParse[nDocs];
         for (int i = 0; i < nDocs; i++) {
             XContentBuilder b = XContentFactory.jsonBuilder();
-            b.startObject().field("field", randomValue(8)).endObject();
+            b.startObject().field("field", randomValue(512)).endObject();
             sources[i] = new SourceToParse(UUIDs.randomBase64UUID(), BytesReference.bytes(b), XContentType.JSON);
         }
     }

docs/changelog/119967.yaml

@@ -0,0 +1,5 @@
+pr: 119967
+summary: Add `index_options` to `semantic_text` field mappings
+area: Mapping
+type: enhancement
+issues: [ ]

docs/changelog/127636.yaml

@@ -0,0 +1,17 @@
+pr: 127636
+summary: Disallow mixed quoted/unquoted patterns in FROM
+area: ES|QL
+type: breaking
+issues:
+ - 122651
+breaking:
+  title: Disallow mixed quoted/unquoted patterns in FROM
+  area: ES|QL
+  details: "Previously, the ES|QL grammar allowed users to individually quote constituent strings in index patterns\
+    \ such as \"remote_cluster\":\"index_name\". This would allow users to write complex malformed index patterns\
+    \ that often slip through grammar and the subsequent validation. This could result in runtime errors\
+    \ that can be misleading. This change simplifies the grammar to early reject such malformed index patterns\
+    \ at the parsing stage, allowing users to write simpler queries and see more relevant and meaningful\
+    \ errors."
+  impact: "Users can write queries with simpler index patterns and see more meaningful and relevant errors."
+  notable: false

docs/changelog/129507.yaml

@@ -0,0 +1,6 @@
+pr: 129507
+summary: Using a temp `IndexService` for template validation
+area: Indices APIs
+type: bug
+issues:
+ - 129473

docs/changelog/129509.yaml

@@ -0,0 +1,6 @@
+pr: 129509
+summary: Fix NPE in `SemanticTextHighlighter`
+area: Search
+type: bug
+issues:
+ - 129501

docs/changelog/129548.yaml

@@ -0,0 +1,5 @@
+pr: 129548
+summary: Fix NPE in `flat_bbq` scorer when all vectors are missing
+area: Vector Search
+type: bug
+issues: []

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -28,7 +28,7 @@ service.
 
 Using `semantic_text`, you won’t need to specify how to generate embeddings for
 your data, or how to index it. The {{infer}} endpoint automatically determines
-the embedding generation, indexing, and query to use. 
+the embedding generation, indexing, and query to use.
 Newly created indices with `semantic_text` fields using dense embeddings will be
 [quantized](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization)
 to `bbq_hnsw` automatically.
@@ -111,6 +111,33 @@ the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/ope
 to create the endpoint. If not specified, the {{infer}} endpoint defined by
 `inference_id` will be used at both index and query time.
 
+`index_options`
+:   (Optional, string) Specifies the index options to override default values
+for the field. Currently, `dense_vector` index options are supported.
+For text embeddings, `index_options` may match any allowed
+[dense_vector index options](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-index-options).
+
+An example of how to set index_options for a `semantic_text` field:
+
+```console
+PUT my-index-000004
+{
+  "mappings": {
+    "properties": {
+      "inference_field": {
+        "type": "semantic_text",
+        "inference_id": "my-text-embedding-endpoint",
+        "index_options": {
+          "dense_vector": {
+            "type": "int4_flat"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
 `chunking_settings`
 :   (Optional, object) Settings for chunking text into smaller passages.
 If specified, these will override the chunking settings set in the {{infer-cap}}
@@ -138,8 +165,10 @@ To completely disable chunking, use the `none` chunking strategy.
     or `1`. Required for `sentence` type chunking settings
 
 ::::{warning}
-If the input exceeds the maximum token limit of the underlying model,  some services (such as OpenAI) may return an 
-error. In contrast, the `elastic` and `elasticsearch` services  will automatically truncate the input to fit within the 
+If the input exceeds the maximum token limit of the underlying model, some
+services (such as OpenAI) may return an
+error. In contrast, the `elastic` and `elasticsearch` services will
+automatically truncate the input to fit within the
 model's limit.
 ::::
 
@@ -173,7 +202,8 @@ For more details on chunking and how to configure chunking settings,
 see [Configuring chunking](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-inference)
 in the Inference API documentation.
 
-You can pre-chunk the input by sending it to Elasticsearch as an array of strings.
+You can pre-chunk the input by sending it to Elasticsearch as an array of
+strings.
 Example:
 
 ```console
@@ -203,15 +233,20 @@ PUT test-index/_doc/1
 ```
 
 1. The text is pre-chunked and provided as an array of strings.
-   Each element in the array represents a single chunk that will be sent directly to the inference service without further chunking.
+   Each element in the array represents a single chunk that will be sent
+   directly to the inference service without further chunking.
 
 **Important considerations**:
 
-* When providing pre-chunked input, ensure that you set the chunking strategy to `none` to avoid additional processing.
-* Each chunk should be sized carefully, staying within the token limit of the inference service and the underlying model.
-* If a chunk exceeds the model's token limit, the behavior depends on the service:
-  * Some services (such as OpenAI) will return an error.
-  * Others (such as `elastic` and `elasticsearch`) will automatically truncate the input.
+* When providing pre-chunked input, ensure that you set the chunking strategy to
+  `none` to avoid additional processing.
+* Each chunk should be sized carefully, staying within the token limit of the
+  inference service and the underlying model.
+* If a chunk exceeds the model's token limit, the behavior depends on the
+  service:
+    * Some services (such as OpenAI) will return an error.
+    * Others (such as `elastic` and `elasticsearch`) will automatically truncate
+      the input.
 
 Refer
 to [this tutorial](docs-content://solutions/search/semantic-search/semantic-search-semantic-text.md)