Let IgnoredSourceFieldLoader.SFV decode NameValue

Author: martijnvg

server/src/main/java/org/elasticsearch/index/fieldvisitor/IgnoredSourceFieldLoader.java

@@ -13,24 +13,39 @@
 import org.apache.lucene.index.LeafReader;
 import org.apache.lucene.index.LeafReaderContext;
 import org.apache.lucene.index.StoredFieldVisitor;
-import org.apache.lucene.util.BytesRef;
 import org.elasticsearch.common.CheckedBiConsumer;
 import org.elasticsearch.common.bytes.BytesReference;
 import org.elasticsearch.common.lucene.index.SequentialStoredFieldsLeafReader;
+import org.elasticsearch.index.mapper.FallbackSyntheticSourceBlockLoader;
 import org.elasticsearch.index.mapper.IgnoredSourceFieldMapper;
 import org.elasticsearch.search.fetch.StoredFieldsSpec;
 
 import java.io.IOException;
 import java.util.ArrayList;
+import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.Set;
 
 class IgnoredSourceFieldLoader extends StoredFieldLoader {
 
+    final Set<String> potentialFieldsInIgnoreSource;
+
+    IgnoredSourceFieldLoader(StoredFieldsSpec spec) {
+        Set<String> potentialFieldsInIgnoreSource = new HashSet<>();
+        for (String requiredStoredField : spec.requiredStoredFields()) {
+            if (requiredStoredField.startsWith(IgnoredSourceFieldMapper.NAME)) {
+                String fieldName = requiredStoredField.substring(IgnoredSourceFieldMapper.NAME.length());
+                potentialFieldsInIgnoreSource.addAll(FallbackSyntheticSourceBlockLoader.splitIntoFieldPaths(fieldName));
+            }
+        }
+        this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource;
+    }
+
     @Override
     public LeafStoredFieldLoader getLoader(LeafReaderContext ctx, int[] docs) throws IOException {
         var reader = sequentialReader(ctx);
-        var visitor = new SFV();
+        var visitor = new SFV(potentialFieldsInIgnoreSource);
         return new LeafStoredFieldLoader() {
 
             private int doc = -1;
@@ -73,29 +88,37 @@ public List<String> fieldsToLoad() {
 
     static class SFV extends StoredFieldVisitor {
 
-        boolean processing;
+        boolean done;
         final List<Object> values = new ArrayList<>();
+        final Set<String> potentialFieldsInIgnoreSource;
+
+        SFV(Set<String> potentialFieldsInIgnoreSource) {
+            this.potentialFieldsInIgnoreSource = potentialFieldsInIgnoreSource;
+        }
 
         @Override
         public Status needsField(FieldInfo fieldInfo) throws IOException {
-            if (IgnoredSourceFieldMapper.NAME.equals(fieldInfo.name)) {
-                processing = true;
-                return Status.YES;
-            } else if (processing) {
+            if (done) {
                 return Status.STOP;
+            } else if (IgnoredSourceFieldMapper.NAME.equals(fieldInfo.name)) {
+                return Status.YES;
+            } else {
+                return Status.NO;
             }
-
-            return Status.NO;
         }
 
         @Override
         public void binaryField(FieldInfo fieldInfo, byte[] value) throws IOException {
-            values.add(new BytesRef(value));
+            var result = IgnoredSourceFieldMapper.decodeIfMatch(value, potentialFieldsInIgnoreSource);
+            if (result != null) {
+                done = true;
+                values.add(result);
+            }
         }
 
         void reset() {
             values.clear();
-            processing = false;
+            done = false;
         }
 
     }
@@ -104,7 +127,7 @@ static boolean supports(StoredFieldsSpec spec) {
         return spec.requiresSource() == false
             && spec.requiresMetadata() == false
             && spec.requiredStoredFields().size() == 1
-            && spec.requiredStoredFields().contains(IgnoredSourceFieldMapper.NAME);
+            && spec.requiredStoredFields().iterator().next().startsWith(IgnoredSourceFieldMapper.NAME);
     }
 
     // TODO: use provided one

server/src/main/java/org/elasticsearch/index/fieldvisitor/StoredFieldLoader.java

@@ -51,7 +51,7 @@ public static StoredFieldLoader fromSpec(StoredFieldsSpec spec) {
             return StoredFieldLoader.empty();
         }
         if (IgnoredSourceFieldLoader.supports(spec)) {
-            return new IgnoredSourceFieldLoader();
+            return new IgnoredSourceFieldLoader(spec);
         }
         return create(spec.requiresSource(), spec.requiredStoredFields());
     }
@@ -95,7 +95,7 @@ public static StoredFieldLoader fromSpecSequential(StoredFieldsSpec spec) {
             return StoredFieldLoader.empty();
         }
         if (IgnoredSourceFieldLoader.supports(spec)) {
-            return new IgnoredSourceFieldLoader();
+            return new IgnoredSourceFieldLoader(spec);
         }
 
         List<String> fieldsToLoad = fieldsToLoad(spec.requiresSource(), spec.requiredStoredFields());

server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java

@@ -59,7 +59,7 @@ public RowStrideReader rowStrideReader(LeafReaderContext context) throws IOExcep
 
     @Override
     public StoredFieldsSpec rowStrideStoredFieldSpec() {
-        return new StoredFieldsSpec(false, false, Set.of(IgnoredSourceFieldMapper.NAME));
+        return new StoredFieldsSpec(false, false, Set.of(IgnoredSourceFieldMapper.NAME + "." + fieldName));
     }
 
     @Override
@@ -72,7 +72,7 @@ public SortedSetDocValues ordinals(LeafReaderContext context) throws IOException
         throw new UnsupportedOperationException();
     }
 
-    static Set<String> splitIntoFieldPaths(String fieldName) {
+    public static Set<String> splitIntoFieldPaths(String fieldName) {
         var paths = new HashSet<String>();
         paths.add("_doc");
         var current = new StringBuilder();
@@ -107,7 +107,7 @@ public void read(int docId, StoredFields storedFields, Builder builder) throws I
 
             Map<String, List<IgnoredSourceFieldMapper.NameValue>> valuesForFieldAndParents = new HashMap<>();
             for (Object value : ignoredSource) {
-                IgnoredSourceFieldMapper.NameValue nameValue = IgnoredSourceFieldMapper.decode(value);
+                IgnoredSourceFieldMapper.NameValue nameValue = (IgnoredSourceFieldMapper.NameValue) value;
                 if (fieldPaths.contains(nameValue.name())) {
                     valuesForFieldAndParents.computeIfAbsent(nameValue.name(), k -> new ArrayList<>()).add(nameValue);
                 }

server/src/main/java/org/elasticsearch/index/mapper/IgnoredSourceFieldMapper.java

@@ -186,6 +186,20 @@ static NameValue decode(Object field) {
         return new NameValue(name, parentOffset, value, null);
     }
 
+    public static NameValue decodeIfMatch(byte[] bytes, Set<String> potentialFieldsInIgnoreSource) {
+        int encodedSize = ByteUtils.readIntLE(bytes, 0);
+        int nameSize = encodedSize % PARENT_OFFSET_IN_NAME_OFFSET;
+        int parentOffset = encodedSize / PARENT_OFFSET_IN_NAME_OFFSET;
+
+        String name = new String(bytes, 4, nameSize, StandardCharsets.UTF_8);
+        if (potentialFieldsInIgnoreSource.contains(name)) {
+            BytesRef value = new BytesRef(bytes, 4 + nameSize, bytes.length - nameSize - 4);
+            return new NameValue(name, parentOffset, value, null);
+        } else {
+            return null;
+        }
+    }
+
     // In rare cases decoding values stored in this field can fail leading to entire source
     // not being available.
     // We would like to have an option to lose some values in synthetic source