Merge branch 'main' of github.com:elastic/elasticsearch into cs-remove-error-parser

Author: jonathan-buttner

benchmarks/build.gradle

@@ -41,6 +41,7 @@ dependencies {
   }
   api(project(':libs:h3'))
   api(project(':modules:aggregations'))
+  implementation project(':modules:mapper-extras');
   api(project(':x-pack:plugin:esql-core'))
   api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -38,11 +39,16 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 public class MapperServiceFactory {
 
     public static MapperService create(String mappings) {
+        return create(mappings, Collections.emptyList());
+    }
+
+    public static MapperService create(String mappings, List<MapperPlugin> mapperPlugins) {
         Settings settings = Settings.builder()
             .put("index.number_of_replicas", 0)
             .put("index.number_of_shards", 1)
@@ -51,7 +57,7 @@ public static MapperService create(String mappings) {
             .build();
         IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();
         IndexSettings indexSettings = new IndexSettings(meta, settings);
-        MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
+        MapperRegistry mapperRegistry = new IndicesModule(mapperPlugins).getMapperRegistry();
 
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
         BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);

benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/OptimizedTextBenchmark.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
@@ -34,6 +35,7 @@
 import org.openjdk.jmh.infra.Blackhole;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
@@ -66,7 +68,7 @@ public class OptimizedTextBenchmark {
     private SourceToParse[] sources;
 
     private String randomValue(int length) {
-        final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+        final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ";
         Random random = new Random();
         StringBuilder builder = new StringBuilder(length);
         for (int i = 0; i < length; i++) {
@@ -83,17 +85,17 @@ public void setup() throws IOException {
                     "dynamic": false,
                     "properties": {
                         "field": {
-                            "type": "keyword"
+                            "type": "match_only_text"
                         }
                     }
                 }
             }
-            """);
+            """, List.of(new MapperExtrasPlugin()));
 
         sources = new SourceToParse[nDocs];
         for (int i = 0; i < nDocs; i++) {
             XContentBuilder b = XContentFactory.jsonBuilder();
-            b.startObject().field("field", randomValue(8)).endObject();
+            b.startObject().field("field", randomValue(512)).endObject();
             sources[i] = new SourceToParse(UUIDs.randomBase64UUID(), BytesReference.bytes(b), XContentType.JSON);
         }
     }

docs/changelog/119967.yaml

@@ -0,0 +1,5 @@
+pr: 119967
+summary: Add `index_options` to `semantic_text` field mappings
+area: Mapping
+type: enhancement
+issues: [ ]

docs/changelog/127636.yaml

@@ -0,0 +1,17 @@
+pr: 127636
+summary: Disallow mixed quoted/unquoted patterns in FROM
+area: ES|QL
+type: breaking
+issues:
+ - 122651
+breaking:
+  title: Disallow mixed quoted/unquoted patterns in FROM
+  area: ES|QL
+  details: "Previously, the ES|QL grammar allowed users to individually quote constituent strings in index patterns\
+    \ such as \"remote_cluster\":\"index_name\". This would allow users to write complex malformed index patterns\
+    \ that often slip through grammar and the subsequent validation. This could result in runtime errors\
+    \ that can be misleading. This change simplifies the grammar to early reject such malformed index patterns\
+    \ at the parsing stage, allowing users to write simpler queries and see more relevant and meaningful\
+    \ errors."
+  impact: "Users can write queries with simpler index patterns and see more meaningful and relevant errors."
+  notable: false

docs/changelog/129507.yaml

@@ -0,0 +1,6 @@
+pr: 129507
+summary: Using a temp `IndexService` for template validation
+area: Indices APIs
+type: bug
+issues:
+ - 129473

docs/changelog/129509.yaml

@@ -0,0 +1,6 @@
+pr: 129509
+summary: Fix NPE in `SemanticTextHighlighter`
+area: Search
+type: bug
+issues:
+ - 129501

docs/reference/elasticsearch/mapping-reference/semantic-text.md

@@ -28,7 +28,7 @@ service.
 
 Using `semantic_text`, you won’t need to specify how to generate embeddings for
 your data, or how to index it. The {{infer}} endpoint automatically determines
-the embedding generation, indexing, and query to use. 
+the embedding generation, indexing, and query to use.
 Newly created indices with `semantic_text` fields using dense embeddings will be
 [quantized](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-quantization)
 to `bbq_hnsw` automatically.
@@ -111,6 +111,33 @@ the [Create {{infer}} API](https://www.elastic.co/docs/api/doc/elasticsearch/ope
 to create the endpoint. If not specified, the {{infer}} endpoint defined by
 `inference_id` will be used at both index and query time.
 
+`index_options`
+:   (Optional, string) Specifies the index options to override default values
+for the field. Currently, `dense_vector` index options are supported.
+For text embeddings, `index_options` may match any allowed
+[dense_vector index options](/reference/elasticsearch/mapping-reference/dense-vector.md#dense-vector-index-options).
+
+An example of how to set index_options for a `semantic_text` field:
+
+```console
+PUT my-index-000004
+{
+  "mappings": {
+    "properties": {
+      "inference_field": {
+        "type": "semantic_text",
+        "inference_id": "my-text-embedding-endpoint",
+        "index_options": {
+          "dense_vector": {
+            "type": "int4_flat"
+          }
+        }
+      }
+    }
+  }
+}
+```
+
 `chunking_settings`
 :   (Optional, object) Settings for chunking text into smaller passages.
 If specified, these will override the chunking settings set in the {{infer-cap}}
@@ -138,8 +165,10 @@ To completely disable chunking, use the `none` chunking strategy.
     or `1`. Required for `sentence` type chunking settings
 
 ::::{warning}
-If the input exceeds the maximum token limit of the underlying model,  some services (such as OpenAI) may return an 
-error. In contrast, the `elastic` and `elasticsearch` services  will automatically truncate the input to fit within the 
+If the input exceeds the maximum token limit of the underlying model, some
+services (such as OpenAI) may return an
+error. In contrast, the `elastic` and `elasticsearch` services will
+automatically truncate the input to fit within the
 model's limit.
 ::::
 
@@ -173,7 +202,8 @@ For more details on chunking and how to configure chunking settings,
 see [Configuring chunking](https://www.elastic.co/docs/api/doc/elasticsearch/group/endpoint-inference)
 in the Inference API documentation.
 
-You can pre-chunk the input by sending it to Elasticsearch as an array of strings.
+You can pre-chunk the input by sending it to Elasticsearch as an array of
+strings.
 Example:
 
 ```console
@@ -203,15 +233,20 @@ PUT test-index/_doc/1
 ```
 
 1. The text is pre-chunked and provided as an array of strings.
-   Each element in the array represents a single chunk that will be sent directly to the inference service without further chunking.
+   Each element in the array represents a single chunk that will be sent
+   directly to the inference service without further chunking.
 
 **Important considerations**:
 
-* When providing pre-chunked input, ensure that you set the chunking strategy to `none` to avoid additional processing.
-* Each chunk should be sized carefully, staying within the token limit of the inference service and the underlying model.
-* If a chunk exceeds the model's token limit, the behavior depends on the service:
-  * Some services (such as OpenAI) will return an error.
-  * Others (such as `elastic` and `elasticsearch`) will automatically truncate the input.
+* When providing pre-chunked input, ensure that you set the chunking strategy to
+  `none` to avoid additional processing.
+* Each chunk should be sized carefully, staying within the token limit of the
+  inference service and the underlying model.
+* If a chunk exceeds the model's token limit, the behavior depends on the
+  service:
+    * Some services (such as OpenAI) will return an error.
+    * Others (such as `elastic` and `elasticsearch`) will automatically truncate
+      the input.
 
 Refer
 to [this tutorial](docs-content://solutions/search/semantic-search/semantic-search-semantic-text.md)

docs/reference/elasticsearch/rest-apis/retrieve-selected-fields.md

@@ -17,7 +17,7 @@ By default, each hit in the search response includes the document [`_source`](/r
 You can use both of these methods, though the `fields` option is preferred because it consults both the document data and index mappings. In some instances, you might want to use [other methods](#field-retrieval-methods) of retrieving data.
 
 
-### The `fields` option [search-fields-param]
+## The `fields` option [search-fields-param]
 
 To retrieve specific fields in the search response, use the `fields` parameter. Because it consults the index mappings, the `fields` parameter provides several advantages over referencing the `_source` directly. Specifically, the `fields` parameter:
 
@@ -33,7 +33,7 @@ Other mapping options are also respected, including [`ignore_above`](/reference/
 The `fields` option returns values in the way that matches how {{es}} indexes them. For standard fields, this means that the `fields` option looks in `_source` to find the values, then parses and formats them using the mappings. Selected fields that can’t be found in `_source` are skipped.
 
 
-#### Retrieve specific fields [search-fields-request]
+### Retrieve specific fields [search-fields-request]
 
 The following search request uses the `fields` parameter to retrieve values for the `user.id` field, all fields starting with `http.response.`, and the `@timestamp` field.
 
@@ -69,7 +69,7 @@ By default, document metadata fields like `_id` or `_index` are not returned whe
 
 
 
-#### Response always returns an array [search-fields-response]
+### Response always returns an array [search-fields-response]
 
 The `fields` response always returns an array of values for each field, even when there is a single value in the `_source`. This is because {{es}} has no dedicated array type, and any field could contain multiple values. The `fields` parameter also does not guarantee that array values are returned in a specific order. See the mapping documentation on [arrays](/reference/elasticsearch/mapping-reference/array.md) for more background.
 
@@ -109,7 +109,7 @@ The response includes values as a flat list in the `fields` section for each hit
 ```
 
 
-#### Retrieve nested fields [search-fields-nested]
+### Retrieve nested fields [search-fields-nested]
 
 ::::{dropdown}
 The `fields` response for [`nested` fields](/reference/elasticsearch/mapping-reference/nested.md) is slightly different from that of regular object fields. While leaf values inside regular `object` fields are returned as a flat list, values inside `nested` fields are grouped to maintain the independence of each object inside the original nested array. For each entry inside a nested field array, values are again returned as a flat list unless there are other `nested` fields inside the parent nested object, in which case the same procedure is repeated again for the deeper nested fields.
@@ -246,7 +246,7 @@ However, when the `fields` pattern targets the nested `user` field directly, no
 
 
 
-#### Retrieve unmapped fields [retrieve-unmapped-fields]
+### Retrieve unmapped fields [retrieve-unmapped-fields]
 
 ::::{dropdown}
 By default, the `fields` parameter returns only values of mapped fields. However, {{es}} allows storing fields in `_source` that are unmapped, such as setting [dynamic field mapping](docs-content://manage-data/data-store/mapping/dynamic-field-mapping.md) to `false` or by using an object field with `enabled: false`. These options disable parsing and indexing of the object content.
@@ -326,7 +326,7 @@ The response will contain field results under the  `session_data.object.*` path,
 
 
 
-#### Ignored field values [ignored-field-values]
+### Ignored field values [ignored-field-values]
 
 ::::{dropdown}
 The `fields` section of the response only returns values that were valid when indexed. If your search request asks for values from a field that ignored certain values because they were malformed or too large these values are returned separately in an `ignored_field_values` section.
@@ -578,6 +578,7 @@ Also only leaf fields can be returned via the `stored_fields` option. If an obje
 On its own, `stored_fields` cannot be used to load fields in nested objects — if a field contains a nested object in its path, then no data will be returned for that stored field. To access nested fields, `stored_fields` must be used within an [`inner_hits`](/reference/elasticsearch/rest-apis/retrieve-inner-hits.md) block.
 ::::
 
+For an example that uses the `stored_fields` parameter, refer to [](retrieve-stored-fields.md). 
 
 
 ##### Disable stored fields [disable-stored-fields]

docs/reference/elasticsearch/rest-apis/retrieve-stored-fields.md

@@ -0,0 +1,84 @@
+---
+navigation_title: "Retrieve stored fields"
+mapped_pages:
+  - https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html
+applies_to:
+  stack: all
+---
+
+# Retrieve stored fields using the Get document API [get-stored-fields]
+
+Use the `stored_fields` query parameter in a [Get document](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get) API request to retrieve fields marked as stored (`"store": true`) in the index mapping.
+
+Fields not marked as stored are excluded from the response, even if specified in the request.
+
+::::{tip}
+In most cases, the [`fields`](retrieve-selected-fields.md#search-fields-param) and [`_source`](retrieve-selected-fields.md#source-filtering) parameters produce better results than `stored_fields`.
+::::
+
+For example, these PUT requests define a stored field in the mapping and add a document:
+
+```console
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "counter": {
+        "type": "integer",
+        "store": false
+      },
+      "tags": {
+        "type": "keyword",
+        "store": true
+      }
+    }
+  }
+}
+```
+
+```console
+PUT my-index-000001/_doc/1
+{
+  "counter": 1,
+  "tags": [ "production" ]
+}
+```
+
+% TEST[continued]
+
+This request retrieves the stored fields from the document:
+
+```console
+GET my-index-000001/_doc/1?stored_fields=tags,counter
+```
+
+% TEST[continued]
+
+The API returns the following response:
+
+```console-result
+{
+  "_index": "my-index-000001",
+  "_id": "1",
+  "_version": 1,
+  "_seq_no": 22,
+  "_primary_term": 1,
+  "found": true,
+  "fields": {
+    "tags": [
+      "production"
+    ]
+  }
+}
+```
+
+% TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
+
+Although the `counter` field is specified in the request, it's not included in the response because it's not actually a stored field.
+
+Field values are returned as an array.
+
+::::{note}
+Only leaf fields can be retrieved with the `stored_fields` parameter. If you specify an object field instead, an error is returned.
+::::
+