Second parsing pass tracks array scopes properly (#114621)

Author: kkrik-es

muted-tests.yml

@@ -372,15 +372,6 @@ tests:
 - class: org.elasticsearch.backwards.MixedClusterClientYamlTestSuiteIT
   method: test {p0=synonyms/60_synonym_rule_get/Synonym rule not found}
   issue: https://github.com/elastic/elasticsearch/issues/114444
-- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
-  method: testTermsAggregation
-  issue: https://github.com/elastic/elasticsearch/issues/114554
-- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
-  method: testTermsQuery
-  issue: https://github.com/elastic/elasticsearch/issues/114563
-- class: org.elasticsearch.datastreams.logsdb.qa.LogsDbVersusLogsDbReindexedIntoStandardModeChallengeRestIT
-  method: testMatchAllQuery
-  issue: https://github.com/elastic/elasticsearch/issues/114607
 - class: org.elasticsearch.xpack.esql.optimizer.PhysicalPlanOptimizerTests
   method: testPushSpatialIntersectsEvalToSource {default}
   issue: https://github.com/elastic/elasticsearch/issues/114627

server/src/main/java/org/elasticsearch/index/mapper/DocumentParser.java

@@ -208,6 +208,7 @@ private static List<IgnoredSourceFieldMapper.NameValue> parseDocForMissingValues
         XContentParser parser = context.parser();
         XContentParser.Token currentToken = parser.nextToken();
         List<String> path = new ArrayList<>();
+        List<Boolean> isObjectInPath = new ArrayList<>();  // Tracks if path components correspond to an object or an array.
         String fieldName = null;
         while (currentToken != null) {
             while (currentToken != XContentParser.Token.FIELD_NAME) {
@@ -218,11 +219,16 @@ private static List<IgnoredSourceFieldMapper.NameValue> parseDocForMissingValues
                         parser.skipChildren();
                     } else {
                         path.add(fieldName);
+                        isObjectInPath.add(currentToken == XContentParser.Token.START_OBJECT);
                     }
                     fieldName = null;
                 } else if (currentToken == XContentParser.Token.END_OBJECT || currentToken == XContentParser.Token.END_ARRAY) {
-                    if (currentToken == XContentParser.Token.END_OBJECT && path.isEmpty() == false) {
+                    // Remove the path, if the scope type matches the one when the path was added.
+                    if (isObjectInPath.isEmpty() == false
+                        && (isObjectInPath.getLast() && currentToken == XContentParser.Token.END_OBJECT
+                            || isObjectInPath.getLast() == false && currentToken == XContentParser.Token.END_ARRAY)) {
                         path.removeLast();
+                        isObjectInPath.removeLast();
                     }
                     fieldName = null;
                 }
@@ -237,7 +243,6 @@ private static List<IgnoredSourceFieldMapper.NameValue> parseDocForMissingValues
             if (leaf != null) {
                 parser.nextToken();  // Advance the parser to the value to be read.
                 result.add(leaf.cloneWithValue(context.encodeFlattenedToken()));
-                parser.nextToken();  // Skip the token ending the value.
                 fieldName = null;
             }
             currentToken = parser.nextToken();

server/src/test/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapperTests.java

@@ -901,6 +901,37 @@ public void testNestedArray() throws IOException {
         );
     }
 
+    public void testConflictingFieldNameAfterArray() throws IOException {
+        DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
+            b.startObject("path").startObject("properties");
+            {
+                b.startObject("to").startObject("properties");
+                {
+                    b.startObject("id").field("type", "integer").field("synthetic_source_keep", "arrays").endObject();
+                }
+                b.endObject().endObject();
+                b.startObject("id").field("type", "float").endObject();
+            }
+            b.endObject().endObject();
+        })).documentMapper();
+
+        var syntheticSource = syntheticSource(documentMapper, b -> {
+            b.startObject("path");
+            {
+                b.startArray("to");
+                {
+                    b.startObject().array("id", 1, 20, 3).endObject();
+                    b.startObject().field("id", 10).endObject();
+                }
+                b.endArray();
+                b.field("id", "0.1");
+            }
+            b.endObject();
+        });
+        assertEquals("""
+            {"path":{"id":0.1,"to":{"id":[1,20,3,10]}}}""", syntheticSource);
+    }
+
     public void testArrayWithinArray() throws IOException {
         DocumentMapper documentMapper = createMapperService(syntheticSourceMapping(b -> {
             b.startObject("path");