Fix handling of object arrays FallbackSyntheticSourceBlockLoader (elastic#122528) (elastic#122560)

Author: lkts

server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java

@@ -23,6 +23,7 @@
 import java.util.Map;
 import java.util.Optional;
 import java.util.Set;
+import java.util.Stack;
 
 /**
  * Block loader for fields that use fallback synthetic source implementation.
@@ -191,18 +192,45 @@ private void parseFieldFromParent(IgnoredSourceFieldMapper.NameValue nameValue,
                     .createParser(filterParserConfig, nameValue.value().bytes, nameValue.value().offset + 1, nameValue.value().length - 1)
             ) {
                 parser.nextToken();
-                var fieldNameInParser = new StringBuilder(nameValue.name());
-                while (true) {
-                    if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
-                        fieldNameInParser.append('.').append(parser.currentName());
-                        if (fieldNameInParser.toString().equals(fieldName)) {
-                            parser.nextToken();
-                            break;
+                var fieldNames = new Stack<String>() {
+                    {
+                        push(nameValue.name());
+                    }
+                };
+
+                while (parser.currentToken() != null) {
+                    // We are descending into an object/array hierarchy of arbitrary depth
+                    // until we find the field that we need.
+                    while (true) {
+                        if (parser.currentToken() == XContentParser.Token.FIELD_NAME) {
+                            fieldNames.push(parser.currentName());
+                            var nameInParser = String.join(".", fieldNames);
+                            if (nameInParser.equals(fieldName)) {
+                                parser.nextToken();
+                                break;
+                            }
+                        } else {
+                            assert parser.currentToken() == XContentParser.Token.START_OBJECT
+                                || parser.currentToken() == XContentParser.Token.START_ARRAY;
                         }
+
+                        parser.nextToken();
                     }
+                    parseWithReader(parser, blockValues);
                     parser.nextToken();
+
+                    // We are coming back up in object/array hierarchy.
+                    // If arrays are present we will explore all array items by going back down again.
+                    while (parser.currentToken() == XContentParser.Token.END_OBJECT
+                        || parser.currentToken() == XContentParser.Token.END_ARRAY) {
+                        // When exiting an object arrays we'll see END_OBJECT followed by END_ARRAY, but we only need to pop the object name
+                        // once.
+                        if (parser.currentToken() == XContentParser.Token.END_OBJECT) {
+                            fieldNames.pop();
+                        }
+                        parser.nextToken();
+                    }
                 }
-                parseWithReader(parser, blockValues);
             }
         }
 

test/framework/src/main/java/org/elasticsearch/index/mapper/BlockLoaderTestCase.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.logsdb.datageneration.DataGeneratorSpecification;
 import org.elasticsearch.logsdb.datageneration.DocumentGenerator;
 import org.elasticsearch.logsdb.datageneration.FieldType;
+import org.elasticsearch.logsdb.datageneration.Mapping;
 import org.elasticsearch.logsdb.datageneration.MappingGenerator;
 import org.elasticsearch.logsdb.datageneration.Template;
 import org.elasticsearch.logsdb.datageneration.datasource.DataSourceHandler;
@@ -72,9 +73,13 @@ public DataSourceResponse.ObjectMappingParametersGenerator handle(
 
     public void testBlockLoader() throws IOException {
         var template = new Template(Map.of(fieldName, new Template.Leaf(fieldName, fieldType)));
-        runTest(template, fieldName);
+        var syntheticSource = randomBoolean();
+        var mapping = mappingGenerator.generate(template);
+
+        runTest(template, mapping, syntheticSource, fieldName);
     }
 
+    @SuppressWarnings("unchecked")
     public void testBlockLoaderForFieldInObject() throws IOException {
         int depth = randomIntBetween(0, 3);
 
@@ -94,14 +99,24 @@ public void testBlockLoaderForFieldInObject() throws IOException {
         fullFieldName.append('.').append(fieldName);
         currentLevel.put(fieldName, new Template.Leaf(fieldName, fieldType));
         var template = new Template(top);
-        runTest(template, fullFieldName.toString());
-    }
 
-    private void runTest(Template template, String fieldName) throws IOException {
+        var syntheticSource = randomBoolean();
+
         var mapping = mappingGenerator.generate(template);
+
+        if (syntheticSource && randomBoolean()) {
+            // force fallback synthetic source in the hierarchy
+            var docMapping = (Map<String, Object>) mapping.raw().get("_doc");
+            var topLevelMapping = (Map<String, Object>) ((Map<String, Object>) docMapping.get("properties")).get("top");
+            topLevelMapping.put("synthetic_source_keep", "all");
+        }
+
+        runTest(template, mapping, syntheticSource, fullFieldName.toString());
+    }
+
+    private void runTest(Template template, Mapping mapping, boolean syntheticSource, String fieldName) throws IOException {
         var mappingXContent = XContentBuilder.builder(XContentType.JSON.xContent()).map(mapping.raw());
 
-        var syntheticSource = randomBoolean();
         var mapperService = syntheticSource ? createSytheticSourceMapperService(mappingXContent) : createMapperService(mappingXContent);
 
         var document = documentGenerator.generate(template, mapping);