diff --git a/docs/changelog/144112.yaml b/docs/changelog/144112.yaml new file mode 100644 index 0000000000000..e3ff00a91d8de --- /dev/null +++ b/docs/changelog/144112.yaml @@ -0,0 +1,6 @@ +area: ES|QL +issues: + - 143916 +pr: 144112 +summary: Loading unmapped fields on synthetic `_source` +type: bug diff --git a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java index 9d6865182de3e..6093e63aacc81 100644 --- a/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java +++ b/server/src/main/java/org/elasticsearch/index/mapper/KeywordFieldMapper.java @@ -942,7 +942,9 @@ private FallbackSyntheticSourceBlockLoader.Reader fallbackSyntheticSourceBloc return new FallbackSyntheticSourceBlockLoader.SingleValueReader(nullValueBytes) { @Override public void convertValue(Object value, List accumulator) { - String stringValue = ((BytesRef) value).utf8ToString(); + // When _source is synthetic, unmapped numeric fields are provided as their native Java types (Long, Double, etc.) + // rather than BytesRef. Since we treat all unmapped fields as keyword, we fall back to toString(). + String stringValue = value instanceof BytesRef br ? br.utf8ToString() : value.toString(); String adjusted = applyIgnoreAboveAndNormalizer(stringValue); if (adjusted != null) { // TODO what if the value didn't change? diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java index 55c342d9ed85d..77faf747bc0be 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/generative/GenerativeForkRestTest.java @@ -18,7 +18,7 @@ import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.APPROXIMATION_V5; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.FORK_V9; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.METRICS_GROUP_BY_ALL; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2; +import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.SUBQUERY_IN_FROM_COMMAND; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.UNMAPPED_FIELDS; import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.VIEWS_WITH_BRANCHING; @@ -66,7 +66,7 @@ protected void shouldSkipTest(String testName) throws IOException { // FORK is not supported with unmapped_fields="load", see https://github.com/elastic/elasticsearch/issues/142033 assumeFalse( "FORK is not supported with unmapped_fields=\"load\"", - testCase.requiredCapabilities.contains(OPTIONAL_FIELDS_V2.capabilityName()) + testCase.requiredCapabilities.contains(OPTIONAL_FIELDS_V3.capabilityName()) ); assumeFalse( diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec index 419044aeb717f..b9020b7e666aa 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/flattened.csv-spec @@ -1,6 +1,6 @@ flattened root fields required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -17,7 +17,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened KEEP subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -42,7 +42,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened DROP subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -68,7 +68,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened DROP subfield after KEEP-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -94,7 +94,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with WHERE-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -111,7 +111,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with WHERE and KEEP *-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -129,7 +129,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with WHERE and KEEP specific subfields-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -147,7 +147,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with EVAL TO_UPPER-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -173,7 +173,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with EVAL LENGTH-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -199,7 +199,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed SORT on flattened subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -224,7 +224,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed RENAME flattened subfield-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -250,7 +250,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with STATS count-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -265,7 +265,7 @@ count(*):long | resource.attributes.agent.type:keyword # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with STATS count_distinct-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -280,7 +280,7 @@ count_distinct(resource.attributes.host.name):long # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with INLINE STATS-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs @@ -302,7 +302,7 @@ FROM flattened_otel_logs # Temporarily ignored since loading subfields of flattened type is not allowed flattened subfield with MV_EXPAND-Ignore required_capability: load_flattened_field -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM flattened_otel_logs diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/index/mappings/mapping-k8s-unmapped.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/index/mappings/mapping-k8s-unmapped.json index cb101c6efbfed..9242ee0dabe5e 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/index/mappings/mapping-k8s-unmapped.json +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/index/mappings/mapping-k8s-unmapped.json @@ -4,6 +4,14 @@ "@timestamp": { "type": "date" }, + "cluster": { + "type": "keyword", + "time_series_dimension": true + }, + "pod": { + "type": "keyword", + "time_series_dimension": true + }, "client": { "properties": { "ip": { @@ -31,9 +39,6 @@ }, "network": { "properties": { - "bytes_in": { - "type": "long" - }, "total_bytes_in": { "type": "long", "time_series_metric": "counter", @@ -54,11 +59,6 @@ "up": { "type": "boolean" }, - "tx": { - "type": "integer", - "time_series_metric": "gauge", - "meta": { "unit": "packets" } - }, "rx": { "type": "integer", "time_series_metric": "gauge", diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec index b97f728cb5c04..356b23deb8090 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-load.csv-spec @@ -3,7 +3,7 @@ #################### // These ones verify we don't do anything extra when unmapped_fields is not set to "load" doesNotLoadUnmappedFieldsSort -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 FROM partial_mapping_sample_data | SORT @timestamp DESC @@ -20,7 +20,7 @@ FROM partial_mapping_sample_data ; doesNotLoadUnmappedFieldsStats -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 FROM partial_mapping_sample_data | STATS count(*) BY message @@ -36,7 +36,7 @@ count(*):long | message:keyword ; doesNotLoadUnmappedFieldsInlineStats -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 FROM partial_mapping_sample_data | INLINE STATS count = COUNT(*) BY message @@ -51,7 +51,7 @@ FROM partial_mapping_sample_data ; doesNotLoadUnmappedFieldsInlineStatsNoGrouping -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 FROM partial_mapping_sample_data | INLINE STATS max_duration = MAX(event_duration) @@ -66,7 +66,7 @@ FROM partial_mapping_sample_data ; doesNotLoadUnmappedFieldsEnrich -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: enrich_load FROM partial_mapping_sample_data @@ -84,7 +84,7 @@ FROM partial_mapping_sample_data doesNotLoadUnmappedFieldsLookupJoin-Ignore // temporarily forbidding "load" with LOOKUP JOIN -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: join_lookup_v12 FROM partial_mapping_sample_data @@ -105,7 +105,7 @@ FROM partial_mapping_sample_data ###################### unmappedFieldDoesNotAppearLast -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -118,7 +118,7 @@ FROM partial_mapping_sample_data ; fieldDoesNotExistSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -137,7 +137,7 @@ FROM partial_mapping_sample_data ; fieldIsUnmappedSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -156,7 +156,7 @@ FROM partial_mapping_sample_data ; fieldUnmappedInSourceButExcludedSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -176,7 +176,7 @@ FROM partial_mapping_excluded_source_sample_data ; fieldUnmappedInSourceButSourceDisabledSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -196,7 +196,7 @@ FROM partial_mapping_no_source_sample_data ; statsByUnmappedFieldExistsInSource -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -213,7 +213,7 @@ s:long | c:long | unmapped_message:keyword ; inlineStatsByUnmappedFieldExistsInSource -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -231,7 +231,7 @@ FROM partial_mapping_sample_data ; unmappedFieldNonExistentTsIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -249,8 +249,27 @@ FROM k8s_unmapped 2024-05-10T00:01:25.000Z | qa | null ; +unmappedNumericFromK8sSyntheticTsIndex +required_capability: optional_fields_v3 +required_capability: ts_command_v0 + +SET unmapped_fields="load"\; +TS k8s_unmapped +| KEEP @timestamp, cluster, network.cost +| SORT @timestamp +| LIMIT 5 +; + +@timestamp:date | cluster:keyword | network.cost:keyword +2024-05-10T00:00:29.000Z | staging | 9.375 +2024-05-10T00:00:33.000Z | staging | 1.25 +2024-05-10T00:00:51.000Z | prod | 9.25 +2024-05-10T00:00:57.000Z | prod | 12.125 +2024-05-10T00:01:25.000Z | qa | 5.375 +; + statsImplicitLastOverTimeUnmappedTsIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -268,7 +287,7 @@ time_bucket:datetime | x:keyword ; statsByUnmappedFieldTsIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -281,19 +300,19 @@ TS k8s_unmapped s:long | c:long | event_log:keyword 6 | 1 | Aenean hime -6 | 1 | Cubilia lac -6 | 1 | Lorem ipsum -6 | 1 | Lorem ipsum -6 | 1 | Nunc -6 | 1 | Nunc tortor -6 | 1 | Placerat mi +3 | 1 | Cubilia lac +2 | 1 | Lorem ipsum +3 | 1 | Lorem ipsum +2 | 1 | Nunc +4 | 1 | Nunc tortor +7 | 1 | Placerat mi +4 | 1 | Scelerisque 6 | 1 | Scelerisque -6 | 1 | Scelerisque -6 | 1 | Sed ultrici +1 | 1 | Sed ultrici ; explicitLastOverTimeUnmappedTsIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -305,15 +324,15 @@ TS k8s_unmapped ; cluster:keyword | time_bucket:datetime | x:keyword -prod | 2024-05-10T00:00:00.000Z | vulputate su -prod | 2024-05-10T00:10:00.000Z | vulputate qu +prod | 2024-05-10T00:00:00.000Z | vivamus, soc +prod | 2024-05-10T00:10:00.000Z | vivamus, soc prod | 2024-05-10T00:20:00.000Z | volutpat ves -qa | 2024-05-10T00:00:00.000Z | vulputate su -qa | 2024-05-10T00:10:00.000Z | vulputate qu +qa | 2024-05-10T00:00:00.000Z | vulputate qu +qa | 2024-05-10T00:10:00.000Z | volutpat hac ; fieldIsNestedAndUnmapped -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -332,7 +351,7 @@ FROM partial_mapping_sample_data ; fieldIsNestedAndNonExistent -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -355,7 +374,7 @@ FROM partial_mapping_sample_data ############################### noFieldExistsMultiParametersSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -374,7 +393,7 @@ FROM partial_mapping_sample_data ; mixedFieldsMultiParametersSingleIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -397,7 +416,7 @@ FROM partial_mapping_sample_data ##################### mixedFieldsMultiParametersMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data, sample_data METADATA _index @@ -423,7 +442,7 @@ sample_data | 2023-10-23T12:15:03.360Z | null | Connected ; fieldDoesNotExistMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data, sample_data METADATA _index @@ -449,7 +468,7 @@ sample_data | 2023-10-23T12:15:03.360Z | null ; fieldIsUnmappedMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data, sample_data METADATA _index @@ -475,7 +494,7 @@ FROM partial_mapping_sample_data, sample_data METADATA _index ; fieldIsPartiallyUnmappedMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -501,7 +520,7 @@ sample_data | Connected to 10.1.0.1 ; fieldIsPartiallyUnmappedAndRenamedMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data @@ -528,7 +547,7 @@ Connected to 10.1.0.1 ; fieldIsPartiallyUnmappedPartiallySourceIsDisabledMultiIndex -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -555,7 +574,7 @@ partial_mapping_sample_data | 2024-10-23T13:55:01.543Z | Connected to ; partialMappingStats -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -574,7 +593,7 @@ max(@timestamp):date | count(*):long | message:keyword ; partialMappingCoalesce -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -603,7 +622,7 @@ FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data MET ; partialMappingKeywordCast -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -631,7 +650,7 @@ FROM partial_mapping_sample_data,partial_mapping_excluded_source_sample_data MET ; partialMappingStatsAfterCast -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: source_field_mapping SET unmapped_fields="load"\; @@ -647,7 +666,7 @@ count(*):long | message::INT:integer ; filterOnPartiallyUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -666,7 +685,7 @@ sample_data | Connection error ; sortOnPartiallyUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -684,7 +703,7 @@ no_mapping_sample_data | Connected to 10.1.0.2! ; filterOnFullyUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -700,7 +719,7 @@ FROM partial_mapping_sample_data ; filterOnConjunctionMappedAndUnmapped -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -716,7 +735,7 @@ FROM partial_mapping_sample_data ; filterOnConjunctionMappedAndNonExistent -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -728,7 +747,7 @@ _index:keyword | message:keyword | does_not_exist:keyword ; filterOnDisjunctionMappedAndUnmapped -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -745,7 +764,7 @@ FROM partial_mapping_sample_data ; filterOnDisjunctionMappedAndNonExistent -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -761,7 +780,7 @@ sample_data | Connection error | null ; sortOnNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM no_mapping_sample_data METADATA _index @@ -779,7 +798,7 @@ no_mapping_sample_data | null ; sortOnUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -796,7 +815,7 @@ FROM partial_mapping_sample_data ; sortOnExpressionOnUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -814,7 +833,7 @@ FROM partial_mapping_sample_data ; sortOnMultipleFieldsWithNonExistent -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -832,7 +851,7 @@ no_mapping_sample_data | Connected to 10.1.0.2! | null ; sortOnMultipleFieldsWithUnmapped -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -850,7 +869,7 @@ FROM partial_mapping_sample_data ; filterOnNonExistentFieldCastToLong -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM sample_data, no_mapping_sample_data METADATA _index @@ -862,7 +881,7 @@ _index:keyword | message:keyword | does_not_exist:keyword ; filterOnUnmappedFieldCastToLong -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 SET unmapped_fields="load"\; FROM partial_mapping_sample_data @@ -875,7 +894,7 @@ FROM partial_mapping_sample_data ; maxOverTimeOfKeyword -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: ts_command_v0 SET unmapped_fields="load"\; @@ -897,9 +916,59 @@ us | 2024-05-10T00:04:00.000Z | k8s us | 2024-05-10T00:04:00.000Z | k8s_unmapped ; +// Copied over from max_over_time_with_filtering, except bytes_in is unmapped. +maxOverTimeWithFiltering +required_capability: optional_fields_v3 +required_capability: ts_command_v0 + +SET unmapped_fields="load"\; +TS k8s_unmapped +| WHERE pod == "one" +| STATS tx = sum(max_over_time(network.bytes_in :: long)) BY cluster, time_bucket = bucket(@timestamp, 10minute) +| SORT time_bucket, cluster +| LIMIT 10 +; + +tx:long | cluster:keyword | time_bucket:datetime +970 | prod | 2024-05-10T00:00:00.000Z +842 | qa | 2024-05-10T00:00:00.000Z +753 | staging | 2024-05-10T00:00:00.000Z +990 | prod | 2024-05-10T00:10:00.000Z +1006 | qa | 2024-05-10T00:10:00.000Z +947 | staging | 2024-05-10T00:10:00.000Z +953 | prod | 2024-05-10T00:20:00.000Z +917 | qa | 2024-05-10T00:20:00.000Z +749 | staging | 2024-05-10T00:20:00.000Z +; + +// Copied over from eval_on_max_over_time, except bytes_in is unmapped. +evalOnMaxOverTime +required_capability: optional_fields_v3 +required_capability: ts_command_v0 + +SET unmapped_fields="load"\; +TS k8s_unmapped +| STATS max_bytes = avg(max_over_time(network.bytes_in :: long)) BY cluster, time_bucket = bucket(@timestamp, 10minute) +| EVAL kb_minus_offset = (max_bytes - 100) / 1000.0 +| LIMIT 10 +| SORT time_bucket, cluster +; + +max_bytes:double | cluster:keyword | time_bucket:datetime | kb_minus_offset:double +909.3333333333334 | prod | 2024-05-10T00:00:00.000Z | 0.8093333333333333 +908.6666666666666 | qa | 2024-05-10T00:00:00.000Z | 0.8086666666666666 +794.0 | staging | 2024-05-10T00:00:00.000Z | 0.694 +1005.0 | prod | 2024-05-10T00:10:00.000Z | 0.905 +980.0 | qa | 2024-05-10T00:10:00.000Z | 0.88 +917.6666666666666 | staging | 2024-05-10T00:10:00.000Z | 0.8176666666666667 +846.3333333333334 | prod | 2024-05-10T00:20:00.000Z | 0.7463333333333334 +941.6666666666666 | qa | 2024-05-10T00:20:00.000Z | 0.8416666666666667 +786.0 | staging | 2024-05-10T00:20:00.000Z | 0.686 +; + // https://github.com/elastic/elasticsearch/issues/143991 statsFilteredAggAfterEvalWithDottedUnmappedField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: optional_fields_detect_unmapped_fields_in_agg_filters SET unmapped_fields="load"\; @@ -914,7 +983,7 @@ null | foo // https://github.com/elastic/elasticsearch/issues/143991 statsFilteredAggAfterEvalWithDottedUnmappedFieldLoadsFromSource -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: optional_fields_detect_unmapped_fields_in_agg_filters SET unmapped_fields="load"\; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec index 12916c435a2af..c8e8bc6bdf897 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/unmapped-nullify.csv-spec @@ -86,6 +86,25 @@ nanos:date_nanos 2023-01-23T13:55:01.543123456Z ; +unmappedNumericFromK8sSyntheticTsIndex +required_capability: optional_fields_nullify_tech_preview +required_capability: ts_command_v0 + +SET unmapped_fields="nullify"\; +TS k8s_unmapped +| KEEP @timestamp, cluster, network.cost +| SORT @timestamp +| LIMIT 5 +; + +@timestamp:datetime | cluster:keyword | network.cost:null +2024-05-10T00:00:29.000Z | staging | null +2024-05-10T00:00:33.000Z | staging | null +2024-05-10T00:00:51.000Z | prod | null +2024-05-10T00:00:57.000Z | prod | null +2024-05-10T00:01:25.000Z | qa | null +; + keepStar required_capability: optional_fields_nullify_tech_preview @@ -383,8 +402,8 @@ FROM languages | STATS c = COUNT(*) BY does_not_exist ; -c:long |does_not_exist:null -4 |null +c:long | does_not_exist:null +4 | null ; statsGroupAliasShadowingSourceColumnNoFilter @@ -395,8 +414,8 @@ FROM languages | STATS c = COUNT(*) BY language_code = does_not_exist ; -c:long |language_code:null -4 |null +c:long | language_code:null +4 | null ; statsGroupAliasShadowingSourceColumnWithFilter @@ -408,8 +427,8 @@ FROM languages | STATS c = COUNT(*) BY language_code = does_not_exist, language_name ; -c:long |language_code:null |language_name :keyword -1 |null |English +c:long | language_code:null | language_name :keyword +1 | null | English ; statsGroupAliasShadowingSourceColumnWithFilterAndAggExpression @@ -422,8 +441,8 @@ FROM languages BY language_code = does_not_exist1::INTEGER + does_not_exist2::INTEGER + language_code, language_name ; -c:long |language_code:integer |language_name :keyword -1 |null |English +c:long | language_code:integer | language_name :keyword +1 | null | English ; inlinestatsSum @@ -552,11 +571,11 @@ FROM sample_data, sample_data_str, | SORT client_ip ; -client_ip:ip |foo:null |bar:keyword |baz:null -172.21.0.5 |null |null |null -172.21.2.113 |null |null |null -172.21.2.162 |null |null |null -172.21.3.15 |null |null |null +client_ip:ip | foo:null | bar:keyword | baz:null +172.21.0.5 | null | null | null +172.21.2.113 | null | null | null +172.21.2.162 | null | null | null +172.21.3.15 | null | null | null ; forkBranchesWithDifferentSchemas @@ -795,12 +814,12 @@ TS k8s_unmapped | LIMIT 5 ; -@timestamp:datetime | cluster:null | network.cost:null -2024-05-10T00:00:29.000Z | null | null -2024-05-10T00:00:33.000Z | null | null -2024-05-10T00:00:51.000Z | null | null -2024-05-10T00:00:57.000Z | null | null -2024-05-10T00:01:25.000Z | null | null +@timestamp:datetime | cluster:keyword | network.cost:null +2024-05-10T00:00:29.000Z | staging | null +2024-05-10T00:00:33.000Z | staging | null +2024-05-10T00:00:51.000Z | prod | null +2024-05-10T00:00:57.000Z | prod | null +2024-05-10T00:01:25.000Z | qa | null ; rateOnUnmappedTsIndex @@ -1009,7 +1028,7 @@ FROM employees | EVAL y = coalesce(bar, baz) ; -avg_worked_seconds:long | birth_date:date | emp_no:integer | first_name:keyword | gender:keyword | height:double | height.float:double | height.half_float:double | height.scaled_float:double | hire_date:date | is_rehired:boolean | job_positions:keyword | languages:integer | languages.byte:integer | languages.long:long | languages.short:integer | last_name:keyword | salary:integer | salary_change:double | salary_change.int:integer | salary_change.keyword:keyword | salary_change.long:long | still_hired:boolean | foo:null | bar:null | baz:null | y:null +avg_worked_seconds:long | birth_date:date | emp_no:integer | first_name:keyword | gender:keyword | height:double | height.float:double | height.half_float:double | height.scaled_float:double | hire_date:date | is_rehired:boolean | job_positions:keyword | languages:integer | languages.byte:integer | languages.long:long | languages.short:integer | last_name:keyword | salary:integer | salary_change:double | salary_change.int:integer | salary_change.keyword:keyword | salary_change.long:long | still_hired:boolean | foo:null | bar:null | baz:null | y:null ; @@ -1061,6 +1080,6 @@ ROW a = 12::long | DROP a ; -_fork:keyword | x:long -fork1 | 12 +_fork:keyword | x:long +fork1 | 12 ; diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec index 2f512bff2c3f1..113e072271b0f 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/views.csv-spec @@ -459,7 +459,7 @@ total:long | country:keyword ; unmappedFields_Load_with_Count -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -475,7 +475,7 @@ rehired_count:long ; unmappedFields_Load_with_WithNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -491,7 +491,7 @@ rehired_count:long | null_count:long ; unmappedFields_Load_with_EvalOnNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -509,7 +509,7 @@ c:long ; unmappedFields_Load_with_WhereOnNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -526,7 +526,7 @@ c:long ; unmappedFields_Load_with_StatsByNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: views_with_branching required_capability: views_crud_as_index_actions @@ -542,7 +542,7 @@ c:long | nonexistent_field:keyword ; unmappedFields_Load_with_SortByNonExistentField -required_capability: optional_fields_v2 +required_capability: optional_fields_v3 required_capability: views_with_branching required_capability: views_crud_as_index_actions diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index a62f83eb4db4e..ed41c9cbec744 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -226,7 +226,7 @@ public enum Cap { /** * Support for optional fields (might or might not be present in the mappings) using DEFAULT/NULLIFY only. - * Compared to {@link #OPTIONAL_FIELDS_V2}, this does not enable support for LOAD. + * Compared to {@link #OPTIONAL_FIELDS_V3}, this does not enable support for LOAD. */ OPTIONAL_FIELDS_NULLIFY_TECH_PREVIEW, @@ -249,9 +249,10 @@ public enum Cap { /** * Support for optional fields (might or might not be present in the mappings) using DEFAULT/NULLIFY/LOAD. - * V2: prevent pushing down filters and sorts to Lucene of potentially unmapped fields. + * V2: Prevent pushing down filters and sorts to Lucene of potentially unmapped fields. + * V3: Fix synthetic _source numeric load bug (#143916) */ - OPTIONAL_FIELDS_V2(Build.current().isSnapshot()), + OPTIONAL_FIELDS_V3(Build.current().isSnapshot()), /** * Support specifically for *just* the _index METADATA field. Used by CsvTests, since that is the only metadata field currently @@ -2369,7 +2370,7 @@ public enum Cap { * Reject loading sub-fields of flattened fields when {@code unmapped_fields="load"} * See https://github.com/elastic/elasticsearch/issues/143494 */ - REJECT_LOADING_FLATTENED_SUBFIELDS(OPTIONAL_FIELDS_V2.isEnabled()), + REJECT_LOADING_FLATTENED_SUBFIELDS(OPTIONAL_FIELDS_V3.isEnabled()), FIX_DIV_ERROR_MESSAGE, diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java index 866285829505d..1513dccd8e850 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/QuerySettings.java @@ -96,12 +96,12 @@ public class QuerySettings { String resolution = Foldables.stringLiteralValueOf(value, "Unexpected value"); try { UnmappedResolution res = UnmappedResolution.valueOf(resolution.toUpperCase(Locale.ROOT)); - if (res == UnmappedResolution.LOAD && EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled() == false) { + if (res == UnmappedResolution.LOAD && EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled() == false) { throw new IllegalArgumentException("'LOAD' is only supported in snapshot builds"); } return res; } catch (Exception exc) { - var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled() + var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled() ? UnmappedResolution.values() : Arrays.stream(UnmappedResolution.values()).filter(e -> e != UnmappedResolution.LOAD).toArray(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java index e58ce06e74ce1..b350cceef8b9d 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerUnmappedTests.java @@ -910,7 +910,7 @@ private static String setUnmappedNullify(String query) { } private static String setUnmappedLoad(String query) { - assumeTrue("Requires OPTIONAL_FIELDS_V2", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled()); + assumeTrue("Requires OPTIONAL_FIELDS_V3", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled()); return "SET unmapped_fields=\"load\"; " + query; } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java index cf3dd01f9d741..f2fa46ed8a90c 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/UnmappedGoldenTestCase.java @@ -41,7 +41,7 @@ private Optional tryRunTestsNullifyOnly(String query, EnumSet } private Optional tryRunTestsLoadOnly(String query, EnumSet stages, String... nestedPaths) { - return EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled() + return EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled() ? builder(setUnmappedLoad(query)).nestedPath(ArrayUtils.prepend("load", nestedPaths)).stages(stages).tryRun() : Optional.empty(); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java index 40dbb8f8d72fe..ce2315027aaba 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/parser/SetParserTests.java @@ -199,7 +199,7 @@ private Object settingValue(EsqlStatement query, int position) { } public void testSetUnmappedFields_snapshot() { - assumeTrue("OPTIONAL_FIELDS_V2 option required", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled()); + assumeTrue("OPTIONAL_FIELDS_V3 option required", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled()); var modes = List.of("DEFAULT", "NULLIFY", "LOAD"); verifySetUnmappedFields(modes); @@ -236,7 +236,7 @@ public void testSetUnmappedFieldsWrongValue() { v -> Arrays.stream(UnmappedResolution.values()).anyMatch(x -> x.name().equalsIgnoreCase(v)), () -> randomAlphaOfLengthBetween(0, 10) ); - var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled() + var values = EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled() ? UnmappedResolution.values() : Arrays.stream(UnmappedResolution.values()).filter(e -> e != UnmappedResolution.LOAD).toArray(); expectValidationError( diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java index 58835815fd38f..6c4a18521ef1e 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/plan/QuerySettingsTests.java @@ -114,7 +114,7 @@ public void testValidate_UnmappedFields_techPreview() { } public void testValidate_UnmappedFields_allValues() { - assumeTrue("Requires unmapped fields", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V2.isEnabled()); + assumeTrue("Requires unmapped fields", EsqlCapabilities.Cap.OPTIONAL_FIELDS_V3.isEnabled()); validateUnmappedFields("DEFAULT", "NULLIFY", "LOAD"); }