Alternate approach to speed up ignored_source access (#133839)

In #132142 and #132428 we split up ignored_source entries into distinct
lucene fields, then added an optimized field visitor to speed up
retrieving unmapped values for INSIST_🐔.

However, since this approach creates a unique lucene field for every
ignored_source entry, we can very quickly have a lot of lucene fields
if there are a lot of unique unmapped fields per document. This can
cause significant slowdowns in indexing throughput and merge time.

This PR addresses those limitations by reverting back to keeping all
ignored_source entries under the same lucene field. However, we still
keep some of the speedups from that prior work by continuing to 
coalesce multiple ignored_source entries for the same field into a
single entry, allowing the field visitor to exit early.

Unfortunately, we do lose some time compared to the original
optimizations because now the field visitor cannot look at the
fieldInfo to decide whether or not to visit a field, and it instead
needs to actually visit and materialize each ignored_source entry before
it can decide whether or not to keep it.

Author: jordan-powers

server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java

@@ -14,11 +14,13 @@
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.bytes.BytesReference;
+import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.search.fetch.StoredFieldsSpec;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.Collections;
 import java.util.HashMap;
 import java.util.HashSet;
 import java.util.List;
@@ -38,8 +40,8 @@ class IgnoredSourceFieldLoader extends StoredFieldLoader {
 
         HashMap<String, Set<String>> potentialFieldsInIgnoreSource = new HashMap<>();
         for (String requiredIgnoredField : spec.ignoredFieldsSpec().requiredIgnoredFields()) {
-            for (String potentialStoredField : spec.ignoredFieldsSpec().format().requiredStoredFields(requiredIgnoredField)) {
-                potentialFieldsInIgnoreSource.computeIfAbsent(potentialStoredField, k -> new HashSet<>()).add(requiredIgnoredField);
+            for (String potentialIgnoredField : FallbackSyntheticSourceBlockLoader.splitIntoFieldPaths(requiredIgnoredField)) {
+                potentialFieldsInIgnoreSource.computeIfAbsent(potentialIgnoredField, k -> new HashSet<>()).add(requiredIgnoredField);
             }
         }
         this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource;
@@ -82,7 +84,7 @@ public String routing() {
 
             @Override
             public Map<String, List<Object>> storedFields() {
-                return visitor.values;
+                return Map.of(IgnoredSourceFieldMapper.NAME, Collections.unmodifiableList(visitor.values));
             }
         };
     }
@@ -93,7 +95,7 @@ public List<String> fieldsToLoad() {
     }
 
     static class SFV extends StoredFieldVisitor {
-        final Map<String, List<Object>> values = new HashMap<>();
+        List<List<IgnoredSourceFieldMapper.NameValue>> values = new ArrayList<>();
         final Set<String> fieldNames;
         private final Set<String> unvisitedFields;
         final Map<String, Set<String>> potentialFieldsInIgnoreSource;
@@ -110,18 +112,26 @@ public Status needsField(FieldInfo fieldInfo) throws IOException {
                 return Status.STOP;
             }
 
-            Set<String> foundFields = potentialFieldsInIgnoreSource.get(fieldInfo.name);
-            if (foundFields == null) {
-                return Status.NO;
+            if (fieldInfo.name.equals(IgnoredSourceFieldMapper.NAME)) {
+                return Status.YES;
             }
 
-            unvisitedFields.removeAll(foundFields);
-            return Status.YES;
+            return Status.NO;
         }
 
         @Override
         public void binaryField(FieldInfo fieldInfo, byte[] value) {
-            values.computeIfAbsent(fieldInfo.name, k -> new ArrayList<>()).add(new BytesRef(value));
+            var nameValues = IgnoredSourceFieldMapper.CoalescedIgnoredSourceEncoding.decode(new BytesRef(value));
+            assert nameValues.isEmpty() == false;
+            String fieldPath = nameValues.getFirst().name();
+
+            Set<String> foundValues = potentialFieldsInIgnoreSource.get(fieldPath);
+            if (foundValues == null) {
+                return;
+            }
+
+            unvisitedFields.removeAll(foundValues);
+            values.add(nameValues);
         }
 
         void reset() {
@@ -133,6 +143,6 @@ void reset() {
 
     static boolean supports(StoredFieldsSpec spec) {
         return spec.onlyRequiresIgnoredFields()
-            && spec.ignoredFieldsSpec().format() == IgnoredSourceFieldMapper.IgnoredSourceFormat.PER_FIELD_IGNORED_SOURCE;
+            && spec.ignoredFieldsSpec().format() == IgnoredSourceFieldMapper.IgnoredSourceFormat.COALESCED_SINGLE_IGNORED_SOURCE;
     }
 }

server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java

@@ -9,15 +9,13 @@
 
 package org.elasticsearch.index.fieldvisitor;
 
-import org.apache.lucene.index.FieldInfo;
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.StoredFieldVisitor;
 import org.apache.lucene.index.StoredFields;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
-import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.search.fetch.StoredFieldsSpec;
 
 import java.io.IOException;
@@ -202,29 +200,13 @@ private static class ReaderStoredFieldLoader implements LeafStoredFieldLoader {
         private final CustomFieldsVisitor visitor;
         private int doc = -1;
 
-        private static CustomFieldsVisitor getFieldsVisitor(Set<String> fields, boolean loadSource) {
-            if (fields.contains(IgnoredSourceFieldMapper.NAME)) {
-                return new CustomFieldsVisitor(fields, loadSource) {
-                    @Override
-                    public Status needsField(FieldInfo fieldInfo) {
-                        if (fieldInfo.name.startsWith(IgnoredSourceFieldMapper.NAME)) {
-                            return Status.YES;
-                        }
-                        return super.needsField(fieldInfo);
-                    }
-                };
-            }
-
-            return new CustomFieldsVisitor(fields, loadSource);
-        }
-
         ReaderStoredFieldLoader(
             CheckedBiConsumer<Integer, StoredFieldVisitor, IOException> reader,
             boolean loadSource,
             Set<String> fields
         ) {
             this.reader = reader;
-            this.visitor = getFieldsVisitor(fields, loadSource);
+            this.visitor = new CustomFieldsVisitor(fields, loadSource);
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/get/GetResult.java

@@ -244,7 +244,7 @@ public XContentBuilder toXContentEmbedded(XContentBuilder builder, Params params
 
         for (DocumentField field : metaFields.values()) {
             // TODO: can we avoid having an exception here?
-            if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().startsWith(IgnoredSourceFieldMapper.NAME)) {
+            if (field.getName().equals(IgnoredFieldMapper.NAME) || field.getName().equals(IgnoredSourceFieldMapper.NAME)) {
                 builder.field(field.getName(), field.getValues());
             } else {
                 builder.field(field.getName(), field.<Object>getValue());

server/src/main/java/org/elasticsearch/index/mapper/IgnoredFieldsSpec.java

@@ -13,7 +13,6 @@
 
 import java.util.HashSet;
 import java.util.Set;
-import java.util.stream.Collectors;
 
 /**
  * Defines which fields need to be loaded from _ignored_source during a fetch.
@@ -54,7 +53,6 @@ public IgnoredFieldsSpec merge(IgnoredFieldsSpec other) {
      * Get the set of stored fields required to load the specified fields from _ignored_source.
      */
     public Set<String> requiredStoredFields() {
-        return requiredIgnoredFields.stream().flatMap(field -> format.requiredStoredFields(field).stream()).collect(Collectors.toSet());
-
+        return Set.of(IgnoredSourceFieldMapper.NAME);
     }
 }