Index-time scripted fields to be fetched from doc values

docs/changelog/122376.yaml

@@ -0,0 +1,6 @@
+pr: 122376
+summary: Index-time scripted fields to be fetched from doc values
+area: Search
+type: bug
+issues:
+ - 119939

modules/runtime-fields-common/build.gradle

@@ -22,3 +22,10 @@ dependencies {
   api project(':libs:grok')
   api project(':libs:dissect')
 }
+
+tasks.named("yamlRestCompatTestTransform").configure ({ task ->
+  task.skipTest("runtime_fields/13_keyword_calculated_at_index/fetch fields", "Script no longer executed at fetch time")
+  task.skipTest("runtime_fields/23_long_calculated_at_index/fetch fields", "Script no longer executed at fetch time")
+  task.skipTest("runtime_fields/33_double_calculated_at_index/fetch fields", "Script no longer executed at fetch time")
+  task.skipTest("runtime_fields/103_geo_point_calculated_at_index/fetch fields from source", "Script no longer executed at fetch time")
+})

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/103_geo_point_calculated_at_index.yml

@@ -64,13 +64,25 @@ setup:
   - match: {hits.total.value: 6}
   - match: {hits.hits.0.fields.location.0.type: "Point" }
   - match: {hits.hits.0.fields.location.0.coordinates: [34.89, 13.5] }
-  # calculated from scripts adds annoying extra precision
-  - match: { hits.hits.0.fields.location_from_doc_value.0.type: "Point" }
-  - match: { hits.hits.0.fields.location_from_doc_value.0.coordinates: [ 34.889999935403466, 13.499999991618097 ] }
-  - match: { hits.hits.0.fields.location_from_source.0.type: "Point" }
-  - match: { hits.hits.0.fields.location_from_source.0.coordinates: [ 34.89, 13.5 ] }
+  - match: { hits.hits.0.fields.location_from_doc_value.0: "13.499999991618097, 34.889999935403466" }
+  - match: { hits.hits.0.fields.location_from_source.0: "13.499999991618097, 34.889999935403466" }
 
 ---
+"docvalue_fields":
+  - do:
+      search:
+        index: locations
+        body:
+          sort: timestamp
+          docvalue_fields:
+            - location
+            - location_from_doc_value
+            - location_from_source
+  - match: {hits.total.value: 6}
+  - match: { hits.hits.0.fields.location.0: "13.499999991618097, 34.889999935403466" }
+  - match: { hits.hits.0.fields.location_from_doc_value.0: "13.499999991618097, 34.889999935403466" }
+  - match: { hits.hits.0.fields.location_from_source.0: "13.499999991618097, 34.889999935403466" }
+---
 "exists query":
   - do:
       search:

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/13_keyword_calculated_at_index.yml

@@ -88,7 +88,7 @@ setup:
   - match: {hits.total.value: 6}
   - match: {hits.hits.0.fields.day_of_week: [Thursday] }
   - match: {hits.hits.0.fields.day_of_week_from_source: [Thursday] }
-  - match: {hits.hits.0.fields.day_of_week_letters: [T, h, u, r, s, d, a, y] }
+  - match: {hits.hits.0.fields.day_of_week_letters: [T, a, d, h, r, s, u, y] }
   - match: {hits.hits.0.fields.prefixed_node: [node_c] }
 
 ---

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/23_long_calculated_at_index.yml

@@ -17,6 +17,11 @@ setup:
                 type: double
               node:
                 type: keyword
+              random_long:
+                type: long
+                script:
+                  source: |
+                    emit(new Random().nextLong());
               voltage_times_ten:
                 type: long
                 script:
@@ -90,19 +95,55 @@ setup:
           sort: timestamp
           fields:
             - voltage_times_ten
-            - voltage_times_ten_no_dv
             - temperature_digits
   - match: {hits.total.value: 6}
   - match: {hits.hits.0.fields.voltage_times_ten: [40] }
-  - match: {hits.hits.0.fields.temperature_digits: [2, 0, 2] }
+  - match: {hits.hits.0.fields.temperature_digits: [0, 2, 2] }
   - match: {hits.hits.0.fields.voltage_times_ten: [40] }
-  - match: {hits.hits.0.fields.voltage_times_ten_no_dv: [40] }
   - match: {hits.hits.1.fields.voltage_times_ten: [42] }
   - match: {hits.hits.2.fields.voltage_times_ten: [56] }
   - match: {hits.hits.3.fields.voltage_times_ten: [51] }
   - match: {hits.hits.4.fields.voltage_times_ten: [58] }
   - match: {hits.hits.5.fields.voltage_times_ten: [52] }
 
+---
+"script runs at index time":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          fields:
+            - random_long
+  - match: {hits.total.value: 6}
+  - set: { hits.hits.0.fields.random_long.0: random_value }
+
+  - do:
+      search:
+        index: sensor
+        body:
+          query:
+            match:
+              random_long: $random_value
+          aggs:
+            random_value:
+              terms:
+                field: random_long
+  - match: {hits.total.value: 1}
+  - match: {aggregations.random_value.buckets.0.key: $random_value}
+  - match: {aggregations.random_value.buckets.0.doc_count: 1}
+
+---
+"fetch fields no doc_values":
+  - do:
+      catch: bad_request
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          fields:
+            - voltage_times_ten_no_dv
+
 ---
 "docvalue_fields":
   - do:

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/33_double_calculated_at_index.yml

@@ -17,6 +17,11 @@ setup:
                 type: double
               node:
                 type: keyword
+              random_double:
+                type: double
+                script:
+                  source: |
+                    emit(new Random().nextDouble());
               voltage_percent:
                 type: double
                 script:
@@ -88,18 +93,55 @@ setup:
         index: sensor
         body:
           sort: timestamp
-          fields: [voltage_percent, voltage_percent_no_dv, voltage_sqrts]
+          fields: [voltage_percent, voltage_sqrts]
   - match: {hits.total.value: 6}
   - match: {hits.hits.0.fields.voltage_percent: [0.6896551724137931] }
-  # Scripts that scripts that emit multiple values are supported
-  - match: {hits.hits.0.fields.voltage_sqrts: [4.0, 2.0, 1.4142135623730951] }
+  # Test that scripts that emit multiple values are supported and their results are sorted
+  - match: {hits.hits.0.fields.voltage_sqrts: [1.4142135623730951, 2.0, 4.0] }
   - match: {hits.hits.1.fields.voltage_percent: [0.7241379310344828] }
-  - match: {hits.hits.1.fields.voltage_percent_no_dv: [0.7241379310344828] }
   - match: {hits.hits.2.fields.voltage_percent: [0.9655172413793103] }
   - match: {hits.hits.3.fields.voltage_percent: [0.8793103448275862] }
   - match: {hits.hits.4.fields.voltage_percent: [1.0] }
   - match: {hits.hits.5.fields.voltage_percent: [0.896551724137931] }
 
+---
+"script runs at index time":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          fields:
+            - random_double
+  - match: {hits.total.value: 6}
+  - set: { hits.hits.0.fields.random_double.0: random_value }
+
+  - do:
+      search:
+        index: sensor
+        body:
+          query:
+            match:
+              random_double: $random_value
+          aggs:
+            random_value:
+              terms:
+                field: random_double
+  - match: {hits.total.value: 1}
+  - match: {aggregations.random_value.buckets.0.key: $random_value}
+  - match: {aggregations.random_value.buckets.0.doc_count: 1}
+
+---
+"fetch fields no doc values":
+  - do:
+      catch: bad_request
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          fields:
+            - voltage_percent_no_dv
+
 ---
 "docvalue_fields":
   - do:
@@ -110,7 +152,7 @@ setup:
           docvalue_fields: [voltage_percent, voltage_sqrts]
   - match: {hits.total.value: 6}
   - match: {hits.hits.0.fields.voltage_percent: [0.6896551724137931] }
-  # Scripts that scripts that emit multiple values are supported and their results are sorted
+  # Test that scripts that emit multiple values are supported and their results are sorted
   - match: {hits.hits.0.fields.voltage_sqrts: [1.4142135623730951, 2.0, 4.0] }
   - match: {hits.hits.1.fields.voltage_percent: [0.7241379310344828] }
   - match: {hits.hits.2.fields.voltage_percent: [0.9655172413793103] }

modules/runtime-fields-common/src/yamlRestTest/resources/rest-api-spec/test/runtime_fields/43_date_calculated_at_index.yml

@@ -60,6 +60,11 @@ setup:
                 type: double
               node:
                 type: keyword
+              ingest_time:
+                type: date
+                script:
+                  source: |
+                    emit(new Date().getTime());
 
   - do:
       bulk:
@@ -125,6 +130,27 @@ setup:
         - 2018-01-24T17:41:34.000Z
   - match: {hits.hits.0.fields.formatted_tomorrow: [2018-01-19] }
 
+---
+"script runs at index time":
+  - do:
+      search:
+        index: sensor
+        body:
+          sort: timestamp
+          fields:
+            - ingest_time
+  - match: {hits.total.value: 6}
+  - set: { hits.hits.0.fields.ingest_time.0: ingest_time }
+
+  - do:
+      search:
+        index: sensor
+        body:
+          query:
+            match:
+              ingest_time: $ingest_time
+  - gt: {hits.total.value: 0}
+
 ---
 "docvalue_fields":
   - do:

server/src/main/java/org/elasticsearch/index/mapper/BooleanFieldMapper.java

@@ -40,6 +40,7 @@
 import org.elasticsearch.script.field.BooleanDocValuesField;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.xcontent.XContentBuilder;
@@ -238,7 +239,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
             if (this.scriptValues != null) {
-                return FieldValues.valueFetcher(this.scriptValues, context);
+                return new DocValueFetcher(
+                    docValueFormat(format, null),
+                    context.getForField(this, FielddataOperation.SEARCH),
+                    StoredFieldsSpec.NO_REQUIREMENTS
+                );
             }
             return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet());
         }

server/src/main/java/org/elasticsearch/index/mapper/DateFieldMapper.java

@@ -53,6 +53,7 @@
 import org.elasticsearch.script.field.DateNanosDocValuesField;
 import org.elasticsearch.script.field.ToScriptFieldFactory;
 import org.elasticsearch.search.DocValueFormat;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.runtime.LongScriptFieldDistanceFeatureQuery;
@@ -548,7 +549,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
                 ? DateFormatter.forPattern(format).withLocale(defaultFormatter.locale())
                 : defaultFormatter;
             if (scriptValues != null) {
-                return FieldValues.valueFetcher(scriptValues, v -> format((long) v, formatter), context);
+                return new DocValueFetcher(
+                    docValueFormat(format, null),
+                    context.getForField(this, FielddataOperation.SEARCH),
+                    StoredFieldsSpec.NO_REQUIREMENTS
+                );
             }
             return new SourceValueFetcher(name(), context, nullValue) {
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/GeoPointFieldMapper.java

@@ -27,7 +27,6 @@
 import org.elasticsearch.common.geo.GeoFormatterFactory;
 import org.elasticsearch.common.geo.GeoPoint;
 import org.elasticsearch.common.geo.GeoUtils;
-import org.elasticsearch.common.geo.GeometryFormatterFactory;
 import org.elasticsearch.common.geo.ShapeRelation;
 import org.elasticsearch.common.geo.SimpleVectorTileFormatter;
 import org.elasticsearch.common.unit.DistanceUnit;
@@ -49,6 +48,7 @@
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
 import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.runtime.GeoPointScriptFieldDistanceFeatureQuery;
@@ -419,8 +419,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
             if (scriptValues == null) {
                 return super.valueFetcher(context, format);
             }
-            Function<List<GeoPoint>, List<Object>> formatter = getFormatter(format != null ? format : GeometryFormatterFactory.GEOJSON);
-            return FieldValues.valueListFetcher(scriptValues, formatter, context);
+            return new DocValueFetcher(
+                docValueFormat(format, null),
+                context.getForField(this, FielddataOperation.SEARCH),
+                StoredFieldsSpec.NO_REQUIREMENTS
+            );
         }
 
         @Override

server/src/main/java/org/elasticsearch/index/mapper/IpFieldMapper.java

@@ -41,6 +41,7 @@
 import org.elasticsearch.script.field.IpDocValuesField;
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
 
@@ -285,7 +286,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
             if (scriptValues != null) {
-                return FieldValues.valueFetcher(scriptValues, v -> InetAddresses.toAddrString((InetAddress) v), context);
+                return new DocValueFetcher(
+                    docValueFormat(format, null),
+                    context.getForField(this, FielddataOperation.SEARCH),
+                    StoredFieldsSpec.NO_REQUIREMENTS
+                );
             }
             return new SourceValueFetcher(name(), context, nullValue) {
                 @Override

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -59,6 +59,7 @@
 import org.elasticsearch.script.StringFieldScript;
 import org.elasticsearch.script.field.KeywordDocValuesField;
 import org.elasticsearch.search.aggregations.support.CoreValuesSourceType;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.runtime.StringScriptFieldFuzzyQuery;
@@ -843,7 +844,11 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
             if (this.scriptValues != null) {
-                return FieldValues.valueFetcher(this.scriptValues, context);
+                return new DocValueFetcher(
+                    docValueFormat(format, null),
+                    context.getForField(this, FielddataOperation.SEARCH),
+                    StoredFieldsSpec.NO_REQUIREMENTS
+                );
             }
             return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet());
         }

server/src/main/java/org/elasticsearch/index/mapper/NumberFieldMapper.java

@@ -60,6 +60,7 @@
 import org.elasticsearch.search.DocValueFormat;
 import org.elasticsearch.search.aggregations.support.TimeSeriesValuesSourceType;
 import org.elasticsearch.search.aggregations.support.ValuesSourceType;
+import org.elasticsearch.search.fetch.StoredFieldsSpec;
 import org.elasticsearch.search.lookup.FieldValues;
 import org.elasticsearch.search.lookup.SearchLookup;
 import org.elasticsearch.search.lookup.SourceProvider;
@@ -1965,7 +1966,12 @@ public ValueFetcher valueFetcher(SearchExecutionContext context, String format)
                 throw new IllegalArgumentException("Field [" + name() + "] of type [" + typeName() + "] doesn't support formats.");
             }
             if (this.scriptValues != null) {
-                return FieldValues.valueFetcher(this.scriptValues, context);
+
+                return new DocValueFetcher(
+                    docValueFormat(format, null),
+                    context.getForField(this, FielddataOperation.SEARCH),
+                    StoredFieldsSpec.NO_REQUIREMENTS
+                );
             }
             return sourceValueFetcher(context.isSourceEnabled() ? context.sourcePath(name()) : Collections.emptySet());
         }