Fix ESQL load for unmapped numerics with synthetic source (#143916)

Add forUnmappedLoad flag to KeywordFieldType so only the unmapped-keyword
path (created by ESQL for PotentiallyUnmappedKeywordEsField) converts
decoded Object values to string in the fallback synthetic source reader;
mapped keyword fields keep assuming BytesRef.

Add synthetic-source test cases to unmapped-load and unmapped-nullify:
- partial_mapping_synthetic_sample_data (unmapped_event_duration as long)
- unmappedNumericFromK8sSyntheticTsIndex (TS k8s_unmapped, network.cost)

Made-with: Cursor

Author: GalLalouche

server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java

@@ -556,14 +556,16 @@ public static final class KeywordFieldType extends TextFamilyFieldType {
         private final boolean isDimension;
         private final boolean usesBinaryDocValues;
         private final boolean usesBinaryDocValuesForIgnoredFields;
+        private final boolean forUnmappedLoad;
 
         public KeywordFieldType(
             String name,
             IndexType indexType,
             TextSearchInfo textSearchInfo,
             NamedAnalyzer normalizer,
             Builder builder,
-            boolean isSyntheticSource
+            boolean isSyntheticSource,
+            boolean forUnmappedLoad
         ) {
             super(
                 name,
@@ -587,6 +589,18 @@ public KeywordFieldType(
             this.usesBinaryDocValues = builder.usesBinaryDocValues();
             this.usesBinaryDocValuesForIgnoredFields = builder.indexSettings.getIndexVersionCreated()
                 .onOrAfter(IndexVersions.STORE_IGNORED_KEYWORDS_IN_BINARY_DOC_VALUES);
+            this.forUnmappedLoad = forUnmappedLoad;
+        }
+
+        public KeywordFieldType(
+            String name,
+            IndexType indexType,
+            TextSearchInfo textSearchInfo,
+            NamedAnalyzer normalizer,
+            Builder builder,
+            boolean isSyntheticSource
+        ) {
+            this(name, indexType, textSearchInfo, normalizer, builder, isSyntheticSource, false);
         }
 
         public KeywordFieldType(String name) {
@@ -613,6 +627,7 @@ public KeywordFieldType(
             this.isDimension = false;
             this.usesBinaryDocValues = usesBinaryDocValues;
             this.usesBinaryDocValuesForIgnoredFields = false;
+            this.forUnmappedLoad = false;
         }
 
         public KeywordFieldType(String name, FieldType fieldType, boolean isSyntheticSource) {
@@ -633,6 +648,7 @@ public KeywordFieldType(String name, FieldType fieldType, boolean isSyntheticSou
             this.isDimension = false;
             this.usesBinaryDocValues = false;
             this.usesBinaryDocValuesForIgnoredFields = false;
+            this.forUnmappedLoad = false;
         }
 
         public KeywordFieldType(String name, NamedAnalyzer analyzer) {
@@ -653,6 +669,7 @@ public KeywordFieldType(String name, NamedAnalyzer analyzer) {
             this.isDimension = false;
             this.usesBinaryDocValues = false;
             this.usesBinaryDocValuesForIgnoredFields = false;
+            this.forUnmappedLoad = false;
         }
 
         public boolean usesBinaryDocValues() {
@@ -933,10 +950,20 @@ private FallbackSyntheticSourceBlockLoader.Reader<?> fallbackSyntheticSourceBloc
             return new FallbackSyntheticSourceBlockLoader.SingleValueReader<BytesRef>(nullValueBytes) {
                 @Override
                 public void convertValue(Object value, List<BytesRef> accumulator) {
-                    String stringValue = ((BytesRef) value).utf8ToString();
+                    final String stringValue;
+                    if (forUnmappedLoad) {
+                        if (value == null) {
+                            if (nullValueBytes != null) {
+                                accumulator.add(nullValueBytes);
+                            }
+                            return;
+                        }
+                        stringValue = value instanceof BytesRef br ? br.utf8ToString() : value.toString();
+                    } else {
+                        stringValue = ((BytesRef) value).utf8ToString();
+                    }
                     String adjusted = applyIgnoreAboveAndNormalizer(stringValue);
                     if (adjusted != null) {
-                        // TODO what if the value didn't change?
                         accumulator.add(new BytesRef(adjusted));
                     }
                 }

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java

@@ -108,6 +108,11 @@ public class CsvTestsDataLoader {
         new TestDataset("ul_logs"),
         new TestDataset("sample_data"),
         new TestDataset("partial_mapping_sample_data"),
+        new TestDataset(
+            "partial_mapping_synthetic_sample_data",
+            "mapping-partial_mapping_sample_data.json",
+            "partial_mapping_synthetic_sample_data.csv"
+        ).withSetting("partial-mapping-synthetic-settings.json"),
         new TestDataset("no_mapping_sample_data", "mapping-no_mapping_sample_data.json", "partial_mapping_sample_data.csv").withTypeMapping(
             Stream.of("timestamp", "client_ip", "event_duration").collect(toMap(k -> k, k -> "keyword"))
         ),

x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/partial_mapping_synthetic_sample_data.csv

@@ -0,0 +1,8 @@
+@timestamp:date,client_ip:ip,event_duration:long,message:keyword,unmapped_message:keyword,unmapped_event_duration:long,unmapped.nested:keyword
+2024-10-23T13:55:01.543Z,173.21.3.15,1756466,Connected to 10.1.0.1!,Disconnected from 10.1.0.1,1756468,a
+2024-10-23T13:53:55.832Z,173.21.3.15,5033754,Connection error?,Disconnection error,5033756,b
+2024-10-23T13:52:55.015Z,173.21.3.15,8268152,Connection error?,Disconnection error,8268154,c
+2024-10-23T13:51:54.732Z,173.21.3.15,725447,Connection error?,Disconnection error,725449,d
+2024-10-23T13:33:34.937Z,173.21.0.5,1232381,42,43,1232383,e
+2024-10-23T12:27:28.948Z,173.21.2.113,2764888,Connected to 10.1.0.2!,Disconnected from 10.1.0.2,2764890,f
+2024-10-23T12:15:03.360Z,173.21.2.162,3450232,Connected to 10.1.0.3!,Disconnected from 10.1.0.3,3450234,g

x-pack/plugin/esql/qa/testFixtures/src/main/resources/index/settings/partial-mapping-synthetic-settings.json

@@ -0,0 +1,9 @@
+{
+  "index": {
+    "mapping": {
+      "source": {
+        "mode": "synthetic"
+      }
+    }
+  }
+}

x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec

@@ -195,6 +195,23 @@ FROM partial_mapping_no_source_sample_data
 2024-10-23T12:15:03.360Z | null
 ;
 
+unmappedNumericFromSyntheticSourceSingleIndex
+required_capability: optional_fields_v2
+
+SET unmapped_fields="load"\;
+FROM partial_mapping_synthetic_sample_data
+| KEEP @timestamp, unmapped_event_duration
+| SORT @timestamp DESC
+| LIMIT 4
+;
+
+@timestamp:date          | unmapped_event_duration:keyword
+2024-10-23T13:55:01.543Z | 1756468
+2024-10-23T13:53:55.832Z | 5033756
+2024-10-23T13:52:55.015Z | 8268154
+2024-10-23T13:51:54.732Z | 725449
+;
+
 statsByUnmappedFieldExistsInSource
 required_capability: optional_fields_v2
 
@@ -249,6 +266,25 @@ FROM k8s_unmapped
 2024-05-10T00:01:25.000Z   | qa              | null
 ;
 
+unmappedNumericFromK8sSyntheticTsIndex
+required_capability: optional_fields_v2
+required_capability: ts_command_v0
+
+SET unmapped_fields="load"\;
+TS k8s_unmapped
+| KEEP @timestamp, cluster, network.cost
+| SORT @timestamp
+| LIMIT 5
+;
+
+@timestamp:date            | cluster:keyword | network.cost:keyword
+2024-05-10T00:00:29.000Z   | staging         | 9.375
+2024-05-10T00:00:33.000Z   | staging         | 1.25
+2024-05-10T00:00:51.000Z   | prod            | 9.25
+2024-05-10T00:00:57.000Z   | prod            | 12.125
+2024-05-10T00:01:25.000Z   | qa              | 5.375
+;
+
 statsImplicitLastOverTimeUnmappedTsIndex
 required_capability: optional_fields_v2
 required_capability: ts_command_v0

x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec

@@ -49,6 +49,23 @@ FROM partial_mapping_excluded_source_sample_data
 2024-10-23T12:15:03.360Z | null
 ;
 
+unmappedNumericFromSyntheticSourceSingleIndex
+required_capability: optional_fields_nullify_tech_preview
+
+SET unmapped_fields="nullify"\;
+FROM partial_mapping_synthetic_sample_data
+| KEEP @timestamp, unmapped_event_duration
+| SORT @timestamp DESC
+| LIMIT 4
+;
+
+@timestamp:date          | unmapped_event_duration:null
+2024-10-23T13:55:01.543Z | null
+2024-10-23T13:53:55.832Z | null
+2024-10-23T13:52:55.015Z | null
+2024-10-23T13:51:54.732Z | null
+;
+
 fieldUnmappedInSourceButSourceDisabledSingleIndex
 required_capability: optional_fields_nullify_tech_preview
 required_capability: source_field_mapping
@@ -86,6 +103,25 @@ nanos:date_nanos
 2023-01-23T13:55:01.543123456Z
 ;
 
+unmappedNumericFromK8sSyntheticTsIndex
+required_capability: optional_fields_nullify_tech_preview
+required_capability: ts_command_v0
+
+SET unmapped_fields="nullify"\;
+TS k8s_unmapped
+| KEEP @timestamp, cluster, network.cost
+| SORT @timestamp
+| LIMIT 5
+;
+
+@timestamp:date            | cluster:keyword | network.cost:null
+2024-05-10T00:00:29.000Z   | staging         | null
+2024-05-10T00:00:33.000Z   | staging         | null
+2024-05-10T00:00:51.000Z   | prod            | null
+2024-05-10T00:00:57.000Z   | prod            | null
+2024-05-10T00:01:25.000Z   | qa              | null
+;
+
 keepStar
 required_capability: optional_fields_nullify_tech_preview
 

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/EsPhysicalOperationProviders.java

@@ -325,7 +325,8 @@ static MappedFieldType createUnmappedFieldType(String name, DefaultShardContext
                 new TextSearchInfo(UNMAPPED_FIELD_TYPE, builder.similarity(), Lucene.KEYWORD_ANALYZER, Lucene.KEYWORD_ANALYZER),
                 Lucene.KEYWORD_ANALYZER,
                 builder,
-                context.ctx.isSourceSynthetic()
+                context.ctx.isSourceSynthetic(),
+                true
             );
         }
     }