Fix synthetic source for flattened field when used with ignore_above (elastic#113499) (elastic#113568)

(cherry picked from commit 35fbbec)

# Conflicts:
#	rest-api-spec/build.gradle

Author: lkts

docs/changelog/113499.yaml

@@ -0,0 +1,6 @@
+pr: 113499
+summary: Fix synthetic source for flattened field when used with `ignore_above`
+area: Logs
+type: bug
+issues:
+ - 112044

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/get/100_synthetic_source.yml

@@ -1050,6 +1050,63 @@ flattened field with ignore_above:
 
   - is_false: fields
 
+
+---
+flattened field with ignore_above and arrays:
+  - requires:
+      cluster_features: ["mapper.flattened.ignore_above_with_arrays_support"]
+      reason: requires support of ignore_above synthetic source with arrays
+
+  - do:
+      indices.create:
+        index: test
+        body:
+          mappings:
+            _source:
+              mode: synthetic
+            properties:
+              field:
+                type: flattened
+                ignore_above: 10
+
+  - do:
+      index:
+        index: test
+        id: 1
+        body: |
+          {
+            "field": [
+              { "key1": { "key2": "key2", "key3": "key3_ignored" }, "key4": "key4_ignored", "key5": { "key6": "key6_ignored" }, "key7": "key7" },
+              { "key1": { "key2": "key12", "key13": "key13_ignored" }, "key4": "key14_ignored", "key15": { "key16": "key16_ignored" }, "key17": [ "key17", "key18" ] }
+            ]
+          }
+
+  - do:
+      get:
+        index: test
+        id: 1
+
+  - match: { _index: "test" }
+  - match: { _id: "1" }
+  - match: { _version: 1 }
+  - match: { found: true }
+  - match:
+      _source:
+        field:
+          key1:
+            key2: [ "key12", "key2" ]
+            key3: "key3_ignored"
+            key13: "key13_ignored"
+          key4: [ "key14_ignored", "key4_ignored" ]
+          key5:
+            key6: "key6_ignored"
+          key7: "key7"
+          key15:
+            key16: "key16_ignored"
+          key17: [ "key17", "key18" ]
+
+  - is_false: fields
+
 ---
 completion:
   - requires:

rest-api-spec/src/yamlRestTest/resources/rest-api-spec/test/search/540_ignore_above_synthetic_source.yml

@@ -44,8 +44,8 @@ ignore_above mapping level setting:
 ---
 ignore_above mapping level setting on arrays:
   - requires:
-      cluster_features: [ "mapper.ignore_above_index_level_setting" ]
-      reason: introduce ignore_above index level setting
+      cluster_features: [ "mapper.flattened.ignore_above_with_arrays_support" ]
+      reason: requires support of ignore_above with arrays for flattened fields
   - do:
       indices.create:
         index:  test
@@ -80,9 +80,9 @@ ignore_above mapping level setting on arrays:
             match_all: {}
 
   - length: { hits.hits: 1 }
-  #TODO: synthetic source field reconstruction bug (TBD: add link to the issue here)
+    #TODO: synthetic source field reconstruction bug (TBD: add link to the issue here)
   #- match: { hits.hits.0._source.keyword: ["foo bar", "the quick brown fox"] }
-  - match: { hits.hits.0._source.flattened.value: ["the quick brown fox", "jumps over"] }
+  - match: { hits.hits.0._source.flattened.value: [ "jumps over", "the quick brown fox" ] }
   - match: { hits.hits.0.fields.keyword.0: "foo bar" }
   - match: { hits.hits.0.fields.flattened.0.value: "jumps over" }
 

server/src/main/java/org/elasticsearch/index/mapper/MapperFeatures.java

@@ -44,7 +44,8 @@ public Set<NodeFeature> getFeatures() {
             FlattenedFieldMapper.IGNORE_ABOVE_SUPPORT,
             IndexSettings.IGNORE_ABOVE_INDEX_LEVEL_SETTING,
             SourceFieldMapper.SYNTHETIC_SOURCE_COPY_TO_INSIDE_OBJECTS_FIX,
-            TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF
+            TimeSeriesRoutingHashFieldMapper.TS_ROUTING_HASH_FIELD_PARSES_BYTES_REF,
+            FlattenedFieldMapper.IGNORE_ABOVE_WITH_ARRAYS_SUPPORT
         );
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldMapper.java

@@ -112,9 +112,11 @@
 public final class FlattenedFieldMapper extends FieldMapper {
 
     public static final NodeFeature IGNORE_ABOVE_SUPPORT = new NodeFeature("flattened.ignore_above_support");
+    public static final NodeFeature IGNORE_ABOVE_WITH_ARRAYS_SUPPORT = new NodeFeature("mapper.flattened.ignore_above_with_arrays_support");
 
     public static final String CONTENT_TYPE = "flattened";
     public static final String KEYED_FIELD_SUFFIX = "._keyed";
+    public static final String KEYED_IGNORED_VALUES_FIELD_SUFFIX = "._keyed._ignored";
     public static final String TIME_SERIES_DIMENSIONS_ARRAY_PARAM = "time_series_dimensions";
 
     private static class Defaults {
@@ -835,6 +837,7 @@ private FlattenedFieldMapper(
         this.fieldParser = new FlattenedFieldParser(
             mappedFieldType.name(),
             mappedFieldType.name() + KEYED_FIELD_SUFFIX,
+            mappedFieldType.name() + KEYED_IGNORED_VALUES_FIELD_SUFFIX,
             mappedFieldType,
             builder.depthLimit.get(),
             builder.ignoreAbove.get(),
@@ -903,7 +906,12 @@ public FieldMapper.Builder getMergeBuilder() {
     @Override
     protected SyntheticSourceSupport syntheticSourceSupport() {
         if (fieldType().hasDocValues()) {
-            var loader = new FlattenedSortedSetDocValuesSyntheticFieldLoader(fullPath(), fullPath() + "._keyed", leafName());
+            var loader = new FlattenedSortedSetDocValuesSyntheticFieldLoader(
+                fullPath(),
+                fullPath() + KEYED_FIELD_SUFFIX,
+                ignoreAbove() < Integer.MAX_VALUE ? fullPath() + KEYED_IGNORED_VALUES_FIELD_SUFFIX : null,
+                leafName()
+            );
 
             return new SyntheticSourceSupport.Native(loader);
         }

server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldParser.java

@@ -11,18 +11,15 @@
 
 import org.apache.lucene.document.Field;
 import org.apache.lucene.document.SortedSetDocValuesField;
+import org.apache.lucene.document.StoredField;
 import org.apache.lucene.document.StringField;
 import org.apache.lucene.index.IndexWriter;
 import org.apache.lucene.index.IndexableField;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.xcontent.XContentParserUtils;
 import org.elasticsearch.index.mapper.ContentPath;
 import org.elasticsearch.index.mapper.DocumentParserContext;
-import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.index.mapper.MappedFieldType;
-import org.elasticsearch.index.mapper.XContentDataHelper;
-import org.elasticsearch.xcontent.CopyingXContentParser;
-import org.elasticsearch.xcontent.XContentBuilder;
 import org.elasticsearch.xcontent.XContentParser;
 
 import java.io.IOException;
@@ -39,6 +36,7 @@ class FlattenedFieldParser {
 
     private final String rootFieldFullPath;
     private final String keyedFieldFullPath;
+    private final String keyedIgnoredValuesFieldFullPath;
 
     private final MappedFieldType fieldType;
     private final int depthLimit;
@@ -48,13 +46,15 @@ class FlattenedFieldParser {
     FlattenedFieldParser(
         String rootFieldFullPath,
         String keyedFieldFullPath,
+        String keyedIgnoredValuesFieldFullPath,
         MappedFieldType fieldType,
         int depthLimit,
         int ignoreAbove,
         String nullValue
     ) {
         this.rootFieldFullPath = rootFieldFullPath;
         this.keyedFieldFullPath = keyedFieldFullPath;
+        this.keyedIgnoredValuesFieldFullPath = keyedIgnoredValuesFieldFullPath;
         this.fieldType = fieldType;
         this.depthLimit = depthLimit;
         this.ignoreAbove = ignoreAbove;
@@ -65,36 +65,18 @@ public List<IndexableField> parse(final DocumentParserContext documentParserCont
         XContentParser parser = documentParserContext.parser();
         XContentParserUtils.ensureExpectedToken(XContentParser.Token.START_OBJECT, parser.currentToken(), parser);
 
-        XContentBuilder rawDataForSyntheticSource = null;
-        if (documentParserContext.canAddIgnoredField() && ignoreAbove < Integer.MAX_VALUE) {
-            var copyingParser = new CopyingXContentParser(parser);
-            rawDataForSyntheticSource = copyingParser.getBuilder();
-            parser = copyingParser;
-        }
-
         ContentPath path = new ContentPath();
         List<IndexableField> fields = new ArrayList<>();
 
         var context = new Context(parser, documentParserContext);
         parseObject(context, path, fields);
-        if (rawDataForSyntheticSource != null && context.isIgnoredValueEncountered()) {
-            // One or more inner fields are ignored due to `ignore_above`.
-            // Because of that we will store whole object as is in order to generate synthetic source.
-            documentParserContext.addIgnoredField(
-                IgnoredSourceFieldMapper.NameValue.fromContext(
-                    documentParserContext,
-                    rootFieldFullPath,
-                    XContentDataHelper.encodeXContentBuilder(rawDataForSyntheticSource)
-                )
-            );
-        }
 
         return fields;
     }
 
     private void parseObject(Context context, ContentPath path, List<IndexableField> fields) throws IOException {
         String currentName = null;
-        XContentParser parser = context.getParser();
+        XContentParser parser = context.parser();
         while (true) {
             XContentParser.Token token = parser.nextToken();
             if (token == XContentParser.Token.END_OBJECT) {
@@ -111,7 +93,7 @@ private void parseObject(Context context, ContentPath path, List<IndexableField>
     }
 
     private void parseArray(Context context, ContentPath path, String currentName, List<IndexableField> fields) throws IOException {
-        XContentParser parser = context.getParser();
+        XContentParser parser = context.parser();
         while (true) {
             XContentParser.Token token = parser.nextToken();
             if (token == XContentParser.Token.END_ARRAY) {
@@ -128,7 +110,7 @@ private void parseFieldValue(
         String currentName,
         List<IndexableField> fields
     ) throws IOException {
-        XContentParser parser = context.getParser();
+        XContentParser parser = context.parser();
         if (token == XContentParser.Token.START_OBJECT) {
             path.add(currentName);
             validateDepthLimit(path);
@@ -151,19 +133,23 @@ private void parseFieldValue(
     }
 
     private void addField(Context context, ContentPath path, String currentName, String value, List<IndexableField> fields) {
-        if (value.length() > ignoreAbove) {
-            context.onIgnoredValue();
-            return;
-        }
-
         String key = path.pathAsText(currentName);
         if (key.contains(SEPARATOR)) {
             throw new IllegalArgumentException(
                 "Keys in [flattened] fields cannot contain the reserved character \\0. Offending key: [" + key + "]."
             );
         }
+
         String keyedValue = createKeyedValue(key, value);
         BytesRef bytesKeyedValue = new BytesRef(keyedValue);
+
+        if (value.length() > ignoreAbove) {
+            if (context.documentParserContext().mappingLookup().isSourceSynthetic()) {
+                fields.add(new StoredField(keyedIgnoredValuesFieldFullPath, bytesKeyedValue));
+            }
+            return;
+        }
+
         // check the keyed value doesn't exceed the IndexWriter.MAX_TERM_LENGTH limit enforced by Lucene at index time
         // in that case we can already throw a more user friendly exception here which includes the offending fields key and value lengths
         if (bytesKeyedValue.length > IndexWriter.MAX_TERM_LENGTH) {
@@ -198,10 +184,10 @@ private void addField(Context context, ContentPath path, String currentName, Str
             final String keyedFieldName = FlattenedFieldParser.extractKey(bytesKeyedValue).utf8ToString();
             if (fieldType.isDimension() && fieldType.dimensions().contains(keyedFieldName)) {
                 final BytesRef keyedFieldValue = FlattenedFieldParser.extractValue(bytesKeyedValue);
-                context.getDocumentParserContext()
+                context.documentParserContext()
                     .getDimensions()
                     .addString(rootFieldFullPath + "." + keyedFieldName, keyedFieldValue)
-                    .validate(context.getDocumentParserContext().indexSettings());
+                    .validate(context.documentParserContext().indexSettings());
             }
         }
     }
@@ -239,32 +225,5 @@ static BytesRef extractValue(BytesRef keyedValue) {
         return new BytesRef(keyedValue.bytes, valueStart, keyedValue.length - valueStart);
     }
 
-    private static class Context {
-        private final XContentParser parser;
-        private final DocumentParserContext documentParserContext;
-
-        private boolean ignoredValueEncountered;
-
-        private Context(XContentParser parser, DocumentParserContext documentParserContext) {
-            this.parser = parser;
-            this.documentParserContext = documentParserContext;
-            this.ignoredValueEncountered = false;
-        }
-
-        public XContentParser getParser() {
-            return parser;
-        }
-
-        public DocumentParserContext getDocumentParserContext() {
-            return documentParserContext;
-        }
-
-        public void onIgnoredValue() {
-            this.ignoredValueEncountered = true;
-        }
-
-        public boolean isIgnoredValueEncountered() {
-            return ignoredValueEncountered;
-        }
-    }
+    private record Context(XContentParser parser, DocumentParserContext documentParserContext) {}
 }

server/src/main/java/org/elasticsearch/index/mapper/flattened/FlattenedFieldSyntheticWriterHelper.java

@@ -9,7 +9,6 @@
 
 package org.elasticsearch.index.mapper.flattened;
 
-import org.apache.lucene.index.SortedSetDocValues;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -226,19 +225,23 @@ public boolean equals(Object obj) {
         }
     }
 
-    private final SortedSetDocValues dv;
+    interface SortedKeyedValues {
+        BytesRef next() throws IOException;
+    }
+
+    private final SortedKeyedValues sortedKeyedValues;
 
-    FlattenedFieldSyntheticWriterHelper(final SortedSetDocValues dv) {
-        this.dv = dv;
+    FlattenedFieldSyntheticWriterHelper(final SortedKeyedValues sortedKeyedValues) {
+        this.sortedKeyedValues = sortedKeyedValues;
     }
 
     void write(final XContentBuilder b) throws IOException {
-        KeyValue curr = new KeyValue(dv.lookupOrd(dv.nextOrd()));
+        KeyValue curr = new KeyValue(sortedKeyedValues.next());
         KeyValue prev = KeyValue.EMPTY;
         final List<String> values = new ArrayList<>();
         values.add(curr.value());
-        for (int i = 1; i < dv.docValueCount(); i++) {
-            KeyValue next = new KeyValue(dv.lookupOrd(dv.nextOrd()));
+        for (BytesRef nextValue = sortedKeyedValues.next(); nextValue != null; nextValue = sortedKeyedValues.next()) {
+            KeyValue next = new KeyValue(nextValue);
             writeObject(b, curr, next, curr.start(prev), curr.end(next), values);
             values.add(next.value());
             prev = curr;