experiment: store ignored source as binary doc values.

Author: martijnvg

server/src/main/java/org/elasticsearch/index/IndexVersions.java

@@ -224,6 +224,7 @@ private static Version parseUnchecked(String version) {
     public static final IndexVersion TIME_SERIES_USE_SYNTHETIC_ID_94 = def(9_071_0_00, Version.LUCENE_10_3_2);
     public static final IndexVersion TIME_SERIES_DOC_VALUES_FORMAT_VERSION_3 = def(9_072_0_00, Version.LUCENE_10_3_2);
     public static final IndexVersion STORE_IGNORED_MALFORMED_IN_BINARY_DOC_VALUES = def(9_073_0_00, Version.LUCENE_10_3_2);
+    public static final IndexVersion IGNORED_SOURCE_AS_DOC_VALUES = def(9_074_0_00, Version.LUCENE_10_3_2);
 
     /*
      * STOP! READ THIS FIRST! No, really,

server/src/main/java/org/elasticsearch/index/mapper/FallbackSyntheticSourceBlockLoader.java

@@ -57,14 +57,27 @@ protected FallbackSyntheticSourceBlockLoader(
         this.fieldPaths = splitIntoFieldPaths(fieldName);
     }
 
+    /**
+     * Returns the ignored source format used by this loader.
+     */
+    public IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat() {
+        return ignoredSourceFormat;
+    }
+
     @Override
     public IOFunction<CircuitBreaker, ColumnAtATimeReader> columnAtATimeReader(LeafReaderContext context) {
         return null;
     }
 
     @Override
     public RowStrideReader rowStrideReader(CircuitBreaker breaker, LeafReaderContext context) throws IOException {
-        return new IgnoredSourceRowStrideReader<>(breaker, fieldName, fieldPaths, reader, ignoredSourceFormat);
+        return new IgnoredSourceRowStrideReader<>(
+            breaker,
+            fieldName,
+            fieldPaths,
+            reader,
+            ignoredSourceFormat.createLeafLoader(context.reader())
+        );
     }
 
     @Override
@@ -107,28 +120,29 @@ private static class IgnoredSourceRowStrideReader<T> implements RowStrideReader
         // Contains name of the field and all its parents
         private final Set<String> fieldPaths;
         private final Reader<T> reader;
-        private final IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat;
+        private final IgnoredSourceFieldMapper.IgnoredSourceLeafLoader ignoredSourceLeafLoader;
 
         IgnoredSourceRowStrideReader(
             CircuitBreaker breaker,
             String fieldName,
             Set<String> fieldPaths,
             Reader<T> reader,
-            IgnoredSourceFieldMapper.IgnoredSourceFormat ignoredSourceFormat
+            IgnoredSourceFieldMapper.IgnoredSourceLeafLoader ignoredSourceLeafLoader
         ) {
             breaker.addEstimateBytesAndMaybeBreak(ESTIMATED_SIZE, "load blocks");
             this.breaker = breaker;
             this.fieldName = fieldName;
             this.fieldPaths = fieldPaths;
             this.reader = reader;
-            this.ignoredSourceFormat = ignoredSourceFormat;
+            this.ignoredSourceLeafLoader = ignoredSourceLeafLoader;
         }
 
         @Override
         public void read(int docId, StoredFields storedFields, Builder builder) throws IOException {
-            Map<String, List<IgnoredSourceFieldMapper.NameValue>> valuesForFieldAndParents = ignoredSourceFormat.loadSingleIgnoredField(
+            Map<String, List<IgnoredSourceFieldMapper.NameValue>> valuesForFieldAndParents = ignoredSourceLeafLoader.loadSingleIgnoredField(
                 fieldPaths,
-                storedFields.storedFields()
+                storedFields.storedFields(),
+                docId
             );
 
             if (valuesForFieldAndParents.isEmpty()) {