Alternate approach to speed up ignored_source access

server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java

@@ -14,11 +14,13 @@
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.search.fetch.StoredFieldsSpec;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -38,8 +40,8 @@ class IgnoredSourceFieldLoader extends StoredFieldLoader {
 
         HashMap<String, Set<String>> potentialFieldsInIgnoreSource = new HashMap<>();
         for (String requiredIgnoredField : spec.ignoredFieldsSpec().requiredIgnoredFields()) {
-            for (String potentialStoredField : spec.ignoredFieldsSpec().format().requiredStoredFields(requiredIgnoredField)) {
-                potentialFieldsInIgnoreSource.computeIfAbsent(potentialStoredField, k -> new HashSet<>()).add(requiredIgnoredField);
+            for (String potentialIgnoredField : FallbackSyntheticSourceBlockLoader.splitIntoFieldPaths(requiredIgnoredField)) {
+                potentialFieldsInIgnoreSource.computeIfAbsent(potentialIgnoredField, k -> new HashSet<>()).add(requiredIgnoredField);
             }
         }
         this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource;
@@ -82,7 +84,7 @@ public String routing() {
 
             @Override
             public Map<String, List<Object>> storedFields() {
-                return visitor.values;
+                return Map.of(IgnoredSourceFieldMapper.NAME, Collections.unmodifiableList(visitor.values));
             }
         };
     }
@@ -93,7 +95,7 @@ public List<String> fieldsToLoad() {
     }
 
     static class SFV extends StoredFieldVisitor {
-        final Map<String, List<Object>> values = new HashMap<>();
+        List<List<IgnoredSourceFieldMapper.NameValue>> values = new ArrayList<>();
         final Set<String> fieldNames;
         private final Set<String> unvisitedFields;
         final Map<String, Set<String>> potentialFieldsInIgnoreSource;
@@ -110,18 +112,26 @@ public Status needsField(FieldInfo fieldInfo) throws IOException {
                 return Status.STOP;
             }
 
-            Set<String> foundFields = potentialFieldsInIgnoreSource.get(fieldInfo.name);
-            if (foundFields == null) {
-                return Status.NO;
+            if (fieldInfo.name.equals(IgnoredSourceFieldMapper.NAME)) {
+                return Status.YES;
             }
 
-            unvisitedFields.removeAll(foundFields);
-            return Status.YES;
+            return Status.NO;
         }
 
         @Override
         public void binaryField(FieldInfo fieldInfo, byte[] value) {
-            values.computeIfAbsent(fieldInfo.name, k -> new ArrayList<>()).add(new BytesRef(value));
+            var nameValues = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decode(new BytesRef(value));
+            assert nameValues.isEmpty() == false;
+            String fieldPath = nameValues.getFirst().name();
+
+            Set<String> foundValues = potentialFieldsInIgnoreSource.get(fieldPath);
+            if (foundValues == null) {
+                return;
+            }
+
+            unvisitedFields.removeAll(foundValues);
+            values.add(nameValues);
         }
 
         void reset() {
@@ -133,6 +143,6 @@ void reset() {
 
     static boolean supports(StoredFieldsSpec spec) {
         return spec.onlyRequiresIgnoredFields()
-            && spec.ignoredFieldsSpec().format() == IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE;
+            && spec.ignoredFieldsSpec().format() == IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE;
     }
 }

server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java

@@ -9,15 +9,13 @@
 
 package org.elasticsearch.index.fieldvisitor;
 
-import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.StoredFields;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
-import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.search.fetch.StoredFieldsSpec;
 
 import java.io.IOException;
@@ -202,29 +200,13 @@ private static class ReaderStoredFieldLoader implements LeafStoredFieldLoader {
         private final CustomFieldsVisitor visitor;
         private int doc = -1;
 
-        private static CustomFieldsVisitor getFieldsVisitor(Set<String> fields, boolean loadSource) {
-            if (fields.contains(IgnoredSourceFieldMapper.NAME)) {
-                return new CustomFieldsVisitor(fields, loadSource) {
-                    @Override
-                    public Status needsField(FieldInfo fieldInfo) {
-                        if (fieldInfo.name.startsWith(IgnoredSourceFieldMapper.NAME)) {
-                            return Status.YES;
-                        }
-                        return super.needsField(fieldInfo);
-                    }
-                };
-            }
-
-            return new CustomFieldsVisitor(fields, loadSource);
-        }
-
         ReaderStoredFieldLoader(
             CheckedBiConsumer<Integer, StoredFieldVisitor, IOException> reader,
             boolean loadSource,
             Set<String> fields
         ) {
             this.reader = reader;
-            this.visitor = getFieldsVisitor(fields, loadSource);
+            this.visitor = new CustomFieldsVisitor(fields, loadSource);
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/get/GetResult.java

@@ -244,7 +244,7 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
 
         for (DocumentField field : metaFields.values()) {
             // TODO: can we avoid having an exception here?
-            if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().startsWith(IgnoredSourceFieldMapper.NAME)) {
+            if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().equals(IgnoredSourceFieldMapper.NAME)) {
                 builder.field(field.getName(), field.getValues());
             } else {
                 builder.field(field.getName(), field.<Object>getValue());

server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java

@@ -13,7 +13,6 @@
 
 import java.util.HashSet;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 /**
  * Defines which fields need to be loaded from _ignored_source during a fetch.
@@ -54,7 +53,6 @@ public IgnoredFieldsSpec merge(IgnoredFieldsSpec other) {
      * Get the set of stored fields required to load the specified fields from _ignored_source.
      */
     public Set<String> requiredStoredFields() {
-        return requiredIgnoredFields.stream().flatMap(field -> format.requiredStoredFields(field).stream()).collect(Collectors.toSet());
-
+        return Set.of(IgnoredSourceFieldMapper.NAME);
     }
 }