Merge branch 'main' into data-stream-settings-test-fix

Author: masseyke

benchmarks/build.gradle

@@ -41,6 +41,7 @@ dependencies {
   }
   api(project(':libs:h3'))
   api(project(':modules:aggregations'))
+  implementation project(':modules:mapper-extras');
   api(project(':x-pack:plugin:esql-core'))
   api(project(':x-pack:plugin:core'))
   api(project(':x-pack:plugin:esql'))

benchmarks/src/main/java/org/elasticsearch/benchmark/index/mapper/MapperServiceFactory.java

@@ -29,6 +29,7 @@
 import org.elasticsearch.index.mapper.ProvidedIdFieldMapper;
 import org.elasticsearch.index.similarity.SimilarityService;
 import org.elasticsearch.indices.IndicesModule;
+import org.elasticsearch.plugins.MapperPlugin;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptCompiler;
 import org.elasticsearch.script.ScriptContext;
@@ -38,11 +39,16 @@
 import java.io.IOException;
 import java.io.UncheckedIOException;
 import java.util.Collections;
+import java.util.List;
 import java.util.Map;
 
 public class MapperServiceFactory {
 
     public static MapperService create(String mappings) {
+        return create(mappings, Collections.emptyList());
+    }
+
+    public static MapperService create(String mappings, List<MapperPlugin> mapperPlugins) {
         Settings settings = Settings.builder()
             .put("index.number_of_replicas", 0)
             .put("index.number_of_shards", 1)
@@ -51,7 +57,7 @@ public static MapperService create(String mappings) {
             .build();
         IndexMetadata meta = IndexMetadata.builder("index").settings(settings).build();
         IndexSettings indexSettings = new IndexSettings(meta, settings);
-        MapperRegistry mapperRegistry = new IndicesModule(Collections.emptyList()).getMapperRegistry();
+        MapperRegistry mapperRegistry = new IndicesModule(mapperPlugins).getMapperRegistry();
 
         SimilarityService similarityService = new SimilarityService(indexSettings, null, Map.of());
         BitsetFilterCache bitsetFilterCache = new BitsetFilterCache(indexSettings, BitsetFilterCache.Listener.NOOP);

benchmarks/src/main/java/org/elasticsearch/benchmark/xcontent/OptimizedTextBenchmark.java

@@ -15,6 +15,7 @@
 import org.elasticsearch.common.logging.LogConfigurator;
 import org.elasticsearch.index.mapper.MapperService;
 import org.elasticsearch.index.mapper.SourceToParse;
+import org.elasticsearch.index.mapper.extras.MapperExtrasPlugin;
 import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentFactory;
 import org.elasticsearch.xcontent.XContentType;
@@ -34,6 +35,7 @@
 import org.openjdk.jmh.infra.Blackhole;
 
 import java.io.IOException;
+import java.util.List;
 import java.util.Random;
 import java.util.concurrent.TimeUnit;
 
@@ -66,7 +68,7 @@ public class OptimizedTextBenchmark {
     private SourceToParse[] sources;
 
     private String randomValue(int length) {
-        final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789";
+        final String CHARS = "abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789 ";
         Random random = new Random();
         StringBuilder builder = new StringBuilder(length);
         for (int i = 0; i < length; i++) {
@@ -83,17 +85,17 @@ public void setup() throws IOException {
                     "dynamic": false,
                     "properties": {
                         "field": {
-                            "type": "keyword"
+                            "type": "match_only_text"
                         }
                     }
                 }
             }
-            """);
+            """, List.of(new MapperExtrasPlugin()));
 
         sources = new SourceToParse[nDocs];
         for (int i = 0; i < nDocs; i++) {
             XContentBuilder b = XContentFactory.jsonBuilder();
-            b.startObject().field("field", randomValue(8)).endObject();
+            b.startObject().field("field", randomValue(512)).endObject();
             sources[i] = new SourceToParse(UUIDs.randomBase64UUID(), BytesReference.bytes(b), XContentType.JSON);
         }
     }

docs/changelog/127636.yaml

@@ -0,0 +1,17 @@
+pr: 127636
+summary: Disallow mixed quoted/unquoted patterns in FROM
+area: ES|QL
+type: breaking
+issues:
+ - 122651
+breaking:
+  title: Disallow mixed quoted/unquoted patterns in FROM
+  area: ES|QL
+  details: "Previously, the ES|QL grammar allowed users to individually quote constituent strings in index patterns\
+    \ such as \"remote_cluster\":\"index_name\". This would allow users to write complex malformed index patterns\
+    \ that often slip through grammar and the subsequent validation. This could result in runtime errors\
+    \ that can be misleading. This change simplifies the grammar to early reject such malformed index patterns\
+    \ at the parsing stage, allowing users to write simpler queries and see more relevant and meaningful\
+    \ errors."
+  impact: "Users can write queries with simpler index patterns and see more meaningful and relevant errors."
+  notable: false

docs/reference/elasticsearch/rest-apis/retrieve-selected-fields.md

@@ -17,7 +17,7 @@ By default, each hit in the search response includes the document [`_source`](/r
 You can use both of these methods, though the `fields` option is preferred because it consults both the document data and index mappings. In some instances, you might want to use [other methods](#field-retrieval-methods) of retrieving data.
 
 
-### The `fields` option [search-fields-param]
+## The `fields` option [search-fields-param]
 
 To retrieve specific fields in the search response, use the `fields` parameter. Because it consults the index mappings, the `fields` parameter provides several advantages over referencing the `_source` directly. Specifically, the `fields` parameter:
 
@@ -33,7 +33,7 @@ Other mapping options are also respected, including [`ignore_above`](/reference/
 The `fields` option returns values in the way that matches how {{es}} indexes them. For standard fields, this means that the `fields` option looks in `_source` to find the values, then parses and formats them using the mappings. Selected fields that can’t be found in `_source` are skipped.
 
 
-#### Retrieve specific fields [search-fields-request]
+### Retrieve specific fields [search-fields-request]
 
 The following search request uses the `fields` parameter to retrieve values for the `user.id` field, all fields starting with `http.response.`, and the `@timestamp` field.
 
@@ -69,7 +69,7 @@ By default, document metadata fields like `_id` or `_index` are not returned whe
 
 
 
-#### Response always returns an array [search-fields-response]
+### Response always returns an array [search-fields-response]
 
 The `fields` response always returns an array of values for each field, even when there is a single value in the `_source`. This is because {{es}} has no dedicated array type, and any field could contain multiple values. The `fields` parameter also does not guarantee that array values are returned in a specific order. See the mapping documentation on [arrays](/reference/elasticsearch/mapping-reference/array.md) for more background.
 
@@ -109,7 +109,7 @@ The response includes values as a flat list in the `fields` section for each hit
 ```
 
 
-#### Retrieve nested fields [search-fields-nested]
+### Retrieve nested fields [search-fields-nested]
 
 ::::{dropdown}
 The `fields` response for [`nested` fields](/reference/elasticsearch/mapping-reference/nested.md) is slightly different from that of regular object fields. While leaf values inside regular `object` fields are returned as a flat list, values inside `nested` fields are grouped to maintain the independence of each object inside the original nested array. For each entry inside a nested field array, values are again returned as a flat list unless there are other `nested` fields inside the parent nested object, in which case the same procedure is repeated again for the deeper nested fields.
@@ -246,7 +246,7 @@ However, when the `fields` pattern targets the nested `user` field directly, no
 
 
 
-#### Retrieve unmapped fields [retrieve-unmapped-fields]
+### Retrieve unmapped fields [retrieve-unmapped-fields]
 
 ::::{dropdown}
 By default, the `fields` parameter returns only values of mapped fields. However, {{es}} allows storing fields in `_source` that are unmapped, such as setting [dynamic field mapping](docs-content://manage-data/data-store/mapping/dynamic-field-mapping.md) to `false` or by using an object field with `enabled: false`. These options disable parsing and indexing of the object content.
@@ -326,7 +326,7 @@ The response will contain field results under the  `session_data.object.*` path,
 
 
 
-#### Ignored field values [ignored-field-values]
+### Ignored field values [ignored-field-values]
 
 ::::{dropdown}
 The `fields` section of the response only returns values that were valid when indexed. If your search request asks for values from a field that ignored certain values because they were malformed or too large these values are returned separately in an `ignored_field_values` section.
@@ -578,6 +578,7 @@ Also only leaf fields can be returned via the `stored_fields` option. If an obje
 On its own, `stored_fields` cannot be used to load fields in nested objects — if a field contains a nested object in its path, then no data will be returned for that stored field. To access nested fields, `stored_fields` must be used within an [`inner_hits`](/reference/elasticsearch/rest-apis/retrieve-inner-hits.md) block.
 ::::
 
+For an example that uses the `stored_fields` parameter, refer to [](retrieve-stored-fields.md). 
 
 
 ##### Disable stored fields [disable-stored-fields]

docs/reference/elasticsearch/rest-apis/retrieve-stored-fields.md

@@ -0,0 +1,84 @@
+---
+navigation_title: "Retrieve stored fields"
+mapped_pages:
+  - https://www.elastic.co/guide/en/elasticsearch/reference/current/docs-get.html
+applies_to:
+  stack: all
+---
+
+# Retrieve stored fields using the Get document API [get-stored-fields]
+
+Use the `stored_fields` query parameter in a [Get document](https://www.elastic.co/docs/api/doc/elasticsearch/operation/operation-get) API request to retrieve fields marked as stored (`"store": true`) in the index mapping.
+
+Fields not marked as stored are excluded from the response, even if specified in the request.
+
+::::{tip}
+In most cases, the [`fields`](retrieve-selected-fields.md#search-fields-param) and [`_source`](retrieve-selected-fields.md#source-filtering) parameters produce better results than `stored_fields`.
+::::
+
+For example, these PUT requests define a stored field in the mapping and add a document:
+
+```console
+PUT my-index-000001
+{
+  "mappings": {
+    "properties": {
+      "counter": {
+        "type": "integer",
+        "store": false
+      },
+      "tags": {
+        "type": "keyword",
+        "store": true
+      }
+    }
+  }
+}
+```
+
+```console
+PUT my-index-000001/_doc/1
+{
+  "counter": 1,
+  "tags": [ "production" ]
+}
+```
+
+% TEST[continued]
+
+This request retrieves the stored fields from the document:
+
+```console
+GET my-index-000001/_doc/1?stored_fields=tags,counter
+```
+
+% TEST[continued]
+
+The API returns the following response:
+
+```console-result
+{
+  "_index": "my-index-000001",
+  "_id": "1",
+  "_version": 1,
+  "_seq_no": 22,
+  "_primary_term": 1,
+  "found": true,
+  "fields": {
+    "tags": [
+      "production"
+    ]
+  }
+}
+```
+
+% TESTRESPONSE[s/"_seq_no" : \d+/"_seq_no" : $body._seq_no/ s/"_primary_term" : 1/"_primary_term" : $body._primary_term/]
+
+Although the `counter` field is specified in the request, it's not included in the response because it's not actually a stored field.
+
+Field values are returned as an array.
+
+::::{note}
+Only leaf fields can be retrieved with the `stored_fields` parameter. If you specify an object field instead, an error is returned.
+::::
+

docs/reference/elasticsearch/toc.yml

@@ -93,6 +93,7 @@ toc:
             - file: rest-apis/reindex-indices.md
             - file: rest-apis/retrieve-inner-hits.md
             - file: rest-apis/retrieve-selected-fields.md
+            - file: rest-apis/retrieve-stored-fields.md
             - file: rest-apis/retrievers.md
             - file: rest-apis/search-multiple-data-streams-indices.md
             - file: rest-apis/search-profile.md

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapper.java

@@ -31,6 +31,7 @@
 import org.apache.lucene.util.IOFunction;
 import org.elasticsearch.common.CheckedIntFunction;
 import org.elasticsearch.common.lucene.Lucene;
+import org.elasticsearch.common.text.UTF8DecodingReader;
 import org.elasticsearch.common.unit.Fuzziness;
 import org.elasticsearch.index.IndexVersion;
 import org.elasticsearch.index.analysis.IndexAnalyzers;
@@ -364,7 +365,7 @@ public Query phrasePrefixQuery(TokenStream stream, int slop, int maxExpansions,
         @Override
         public BlockLoader blockLoader(BlockLoaderContext blContext) {
             if (textFieldType.isSyntheticSource()) {
-                return new BlockStoredFieldsReader.BytesFromStringsBlockLoader(storedFieldNameForSyntheticSource());
+                return new BlockStoredFieldsReader.BytesFromBytesRefsBlockLoader(storedFieldNameForSyntheticSource());
             }
             SourceValueFetcher fetcher = SourceValueFetcher.toString(blContext.sourcePaths(name()));
             // MatchOnlyText never has norms, so we have to use the field names field
@@ -385,7 +386,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
                 ) {
                     @Override
                     protected BytesRef storedToBytesRef(Object stored) {
-                        return new BytesRef((String) stored);
+                        return (BytesRef) stored;
                     }
                 };
             }
@@ -443,18 +444,20 @@ public FieldMapper.Builder getMergeBuilder() {
 
     @Override
     protected void parseCreateField(DocumentParserContext context) throws IOException {
-        final String value = context.parser().textOrNull();
+        final var value = context.parser().optimizedTextOrNull();
 
         if (value == null) {
             return;
         }
 
-        Field field = new Field(fieldType().name(), value, fieldType);
+        final var utfBytes = value.bytes();
+        Field field = new Field(fieldType().name(), new UTF8DecodingReader(utfBytes), fieldType);
         context.doc().add(field);
         context.addToFieldNames(fieldType().name());
 
         if (storeSource) {
-            context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), value));
+            final var bytesRef = new BytesRef(utfBytes.bytes(), utfBytes.offset(), utfBytes.length());
+            context.doc().add(new StoredField(fieldType().storedFieldNameForSyntheticSource(), bytesRef));
         }
     }
 
@@ -474,7 +477,7 @@ protected SyntheticSourceSupport syntheticSourceSupport() {
             () -> new StringStoredFieldFieldLoader(fieldType().storedFieldNameForSyntheticSource(), fieldType().name(), leafName()) {
                 @Override
                 protected void write(XContentBuilder b, Object value) throws IOException {
-                    b.value((String) value);
+                    b.value(((BytesRef) value).utf8ToString());
                 }
             }
         );

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/SourceConfirmedTextQuery.java

@@ -41,6 +41,7 @@
 import org.apache.lucene.search.Weight;
 import org.apache.lucene.search.similarities.Similarity;
 import org.apache.lucene.search.similarities.Similarity.SimScorer;
+import org.apache.lucene.util.BytesRef;
 import org.apache.lucene.util.IOFunction;
 import org.elasticsearch.common.CheckedIntFunction;
 import org.elasticsearch.common.lucene.search.MultiPhrasePrefixQuery;
@@ -438,7 +439,13 @@ private MemoryIndex getOrCreateMemoryIndex() throws IOException {
                     if (value == null) {
                         continue;
                     }
-                    cacheEntry.memoryIndex.addField(field, value.toString(), indexAnalyzer);
+                    String valueStr;
+                    if (value instanceof BytesRef valueRef) {
+                        valueStr = valueRef.utf8ToString();
+                    } else {
+                        valueStr = value.toString();
+                    }
+                    cacheEntry.memoryIndex.addField(field, valueStr, indexAnalyzer);
                 }
             }
             return cacheEntry.memoryIndex;

modules/mapper-extras/src/test/java/org/elasticsearch/index/mapper/extras/MatchOnlyTextFieldMapperTests.java

@@ -123,7 +123,12 @@ public void testDefaults() throws IOException {
         ParsedDocument doc = mapper.parse(source(b -> b.field("field", "1234")));
         List<IndexableField> fields = doc.rootDoc().getFields("field");
         assertEquals(1, fields.size());
-        assertEquals("1234", fields.get(0).stringValue());
+
+        var reader = fields.get(0).readerValue();
+        char[] buff = new char[20];
+        assertEquals(4, reader.read(buff));
+        assertEquals("1234", new String(buff, 0, 4));
+
         IndexableFieldType fieldType = fields.get(0).fieldType();
         assertThat(fieldType.omitNorms(), equalTo(true));
         assertTrue(fieldType.tokenized());