ValueFetchers now return a StoredFieldsSpec (#94820)

This allows us to be more conservative about what needs to be loaded
when using the fields API, and opens up the possibility of avoiding
using stored fields or source altogether if we can use doc values to
fetch values.

This commit also uses this new information from ValueFetchers to 
more efficiently preload stored fields for the `fields` API, while
still allowing the lazy loading of individual fields if they are asked
for by scripts or runtime fields which cannot be introspected.

Author: romseygeek

benchmarks/src/main/java/org/elasticsearch/benchmark/script/ScriptScoreBenchmark.java

@@ -58,7 +58,6 @@
 import java.nio.file.Path;
 import java.util.List;
 import java.util.Map;
-import java.util.Set;
 import java.util.concurrent.TimeUnit;
 
 /**
@@ -85,7 +84,6 @@ public class ScriptScoreBenchmark {
         Map.entry("n", new NumberFieldType("n", NumberType.LONG, false, false, true, true, null, Map.of(), null, false, null, null))
     );
     private final IndexFieldDataCache fieldDataCache = new IndexFieldDataCache.None();
-    private final Map<String, Set<String>> sourcePaths = Map.of("n", Set.of("n"));
     private final CircuitBreakerService breakerService = new NoneCircuitBreakerService();
     private final SearchLookup lookup = new SearchLookup(
         fieldTypes::get,

modules/mapper-extras/src/main/java/org/elasticsearch/index/mapper/extras/TokenCountFieldMapper.java

@@ -24,7 +24,6 @@
 import org.elasticsearch.xcontent.XContentBuilder;
 
 import java.io.IOException;
-import java.util.List;
 import java.util.Map;
 import java.util.Objects;
 
@@ -121,7 +120,7 @@ static class TokenCountFieldType extends NumberFieldMapper.NumberFieldType {
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             if (hasDocValues() == false) {
-                return (lookup, doc, ignoredValues) -> List.of();
+                return ValueFetcher.EMPTY;
             }
             return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH));
         }

modules/parent-join/src/main/java/org/elasticsearch/join/mapper/ParentIdFieldMapper.java

@@ -29,7 +29,6 @@
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 
 import java.util.Collections;
-import java.util.List;
 import java.util.Map;
 
 /**
@@ -75,7 +74,7 @@ public IndexFieldData.Builder fielddataBuilder(FieldDataContext fieldDataContext
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             // Although this is an internal field, we return it in the list of all field types. So we
             // provide an empty value fetcher here instead of throwing an error.
-            return (lookup, doc, ignoredValues) -> List.of();
+            return ValueFetcher.EMPTY;
         }
 
         @Override

plugins/mapper-size/src/main/java/org/elasticsearch/index/mapper/size/SizeFieldMapper.java

@@ -21,7 +21,6 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 
 import java.util.Collections;
-import java.util.List;
 
 public class SizeFieldMapper extends MetadataFieldMapper {
     public static final String NAME = "_size";
@@ -57,7 +56,7 @@ private static class SizeFieldType extends NumberFieldType {
         @Override
         public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
             if (hasDocValues() == false) {
-                return (lookup, doc, ignoredValues) -> List.of();
+                return ValueFetcher.EMPTY;
             }
             return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH));
         }

server/src/main/java/org/elasticsearch/index/fieldvisitor/SingleFieldsVisitor.java

@@ -46,7 +46,7 @@ public Status needsField(FieldInfo fieldInfo) {
     }
 
     private void addValue(Object value) {
-        destination.add(field.valueForDisplay(value));
+        destination.add(value);
     }
 
     @Override

server/src/main/java/org/elasticsearch/index/mapper/AbstractScriptFieldType.java

@@ -23,6 +23,7 @@
 import org.elasticsearch.script.CompositeFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.ScriptContext;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.xcontent.XContentBuilder;
 
@@ -176,7 +177,11 @@ protected final void applyScriptContext(SearchExecutionContext context) {
 
     @Override
     public ValueFetcher valueFetcher(SearchExecutionContext context, String format) {
-        return new DocValueFetcher(docValueFormat(format, null), context.getForField(this, FielddataOperation.SEARCH));
+        return new DocValueFetcher(
+            docValueFormat(format, null),
+            context.getForField(this, FielddataOperation.SEARCH),
+            StoredFieldsSpec.NEEDS_SOURCE       // for now we assume runtime fields need source
+        );
     }
 
     /**

server/src/main/java/org/elasticsearch/index/mapper/ArraySourceValueFetcher.java

@@ -10,6 +10,7 @@
 
 import org.elasticsearch.core.Nullable;
 import org.elasticsearch.index.query.SearchExecutionContext;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.Source;
 
 import java.util.ArrayList;
@@ -71,6 +72,11 @@ public List<Object> fetchValues(Source source, int doc, List<Object> ignoredValu
         return values;
     }
 
+    @Override
+    public StoredFieldsSpec storedFieldsSpec() {
+        return StoredFieldsSpec.NEEDS_SOURCE;
+    }
+
     /**
      * Given a value that has been extracted from a document's source, parse it into a standard
      * format. This parsing logic should closely mirror the value parsing in

server/src/main/java/org/elasticsearch/index/mapper/DocValueFetcher.java

@@ -12,6 +12,7 @@
 import org.elasticsearch.index.fielddata.FormattedDocValues;
 import org.elasticsearch.index.fielddata.IndexFieldData;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.Source;
 
 import java.io.IOException;
@@ -23,14 +24,22 @@
 /**
  * Value fetcher that loads from doc values.
  */
+// TODO rename this? It doesn't load from doc values, it loads from fielddata
+// Which might be doc values, but might not be...
 public final class DocValueFetcher implements ValueFetcher {
     private final DocValueFormat format;
     private final IndexFieldData<?> ifd;
     private FormattedDocValues formattedDocValues;
+    private final StoredFieldsSpec storedFieldsSpec;
 
-    public DocValueFetcher(DocValueFormat format, IndexFieldData<?> ifd) {
+    public DocValueFetcher(DocValueFormat format, IndexFieldData<?> ifd, StoredFieldsSpec storedFieldsSpec) {
         this.format = format;
         this.ifd = ifd;
+        this.storedFieldsSpec = storedFieldsSpec;
+    }
+
+    public DocValueFetcher(DocValueFormat format, IndexFieldData<?> ifd) {
+        this(format, ifd, StoredFieldsSpec.NO_REQUIREMENTS);
     }
 
     @Override
@@ -50,4 +59,8 @@ public List<Object> fetchValues(Source source, int doc, List<Object> ignoredValu
         return result;
     }
 
+    @Override
+    public StoredFieldsSpec storedFieldsSpec() {
+        return storedFieldsSpec;
+    }
 }

server/src/main/java/org/elasticsearch/index/mapper/GeoPointScriptFieldType.java

@@ -28,6 +28,7 @@
 import org.elasticsearch.script.GeoPointFieldScript;
 import org.elasticsearch.script.Script;
 import org.elasticsearch.script.field.GeoPointDocValuesField;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.lookup.Source;
 import org.elasticsearch.search.runtime.GeoPointScriptFieldDistanceFeatureQuery;
@@ -215,6 +216,11 @@ public List<Object> fetchValues(Source source, int doc, List<Object> ignoredValu
                 }
                 return formatter.apply(points);
             }
+
+            @Override
+            public StoredFieldsSpec storedFieldsSpec() {
+                return StoredFieldsSpec.NEEDS_SOURCE;
+            }
         };
     }
 }

server/src/main/java/org/elasticsearch/index/mapper/IndexFieldMapper.java

@@ -19,6 +19,7 @@
 import org.elasticsearch.index.query.SearchExecutionContext;
 import org.elasticsearch.script.field.DelegateDocValuesField;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.Source;
 
 import java.util.Collections;
@@ -85,6 +86,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
                 public List<Object> fetchValues(Source source, int doc, List<Object> ignoredValues) {
                     return indexName;
                 }
+
+                @Override
+                public StoredFieldsSpec storedFieldsSpec() {
+                    return StoredFieldsSpec.NO_REQUIREMENTS;
+                }
             };
         }