diff --git a/docs/changelog/133369.yaml b/docs/changelog/133369.yaml new file mode 100644 index 0000000000000..c49924bccef52 --- /dev/null +++ b/docs/changelog/133369.yaml @@ -0,0 +1,5 @@ +pr: 133369 +summary: Enable `date` `date_nanos` implicit casting +area: ES|QL +type: enhancement +issues: [] diff --git a/docs/reference/query-languages/esql/esql-multi-index.md b/docs/reference/query-languages/esql/esql-multi-index.md index b35996329ed05..69f3bfe3d7cc5 100644 --- a/docs/reference/query-languages/esql/esql-multi-index.md +++ b/docs/reference/query-languages/esql/esql-multi-index.md @@ -135,6 +135,62 @@ FROM events_* | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 | | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 | +### Date and date_nanos union type [esql-multi-index-date-date-nanos-union] +```{applies_to} +stack: ga 9.2.0 +``` +When the type of an {{esql}} field is a *union* of `date` and `date_nanos` across different indices, {{esql}} automatically casts all values to the `date_nanos` type during query execution. This implicit casting ensures that all values are handled with nanosecond precision, regardless of their original type. As a result, users can write queries against such fields without needing to perform explicit type conversions, and the query engine will seamlessly align the types for consistent and precise results. + +`date_nanos` fields offer higher precision but have a narrower range of valid values compared to `date` fields. This limits their representable dates roughly from 1970 to 2262. This is because dates are stored as a `long` representing nanoseconds since the epoch. When a field is mapped as both `date` and `date_nanos` across different indices, {{esql}} defaults to the more precise `date_nanos` type. This behavior ensures that no precision is lost when querying multiple indices with differing date field types. For dates that fall outside the valid range of `date_nanos` in fields that are mapped to both `date` and `date_nanos` across different indices, {{esql}} returns null by default. However, users can explicitly cast these fields to the `date` type to obtain a valid value, with precision limited to milliseconds. + +For example, if the `@timestamp` field is mapped as `date` in one index and `date_nanos` in another, {{esql}} will automatically treat all `@timestamp` values as `date_nanos` during query execution. This allows users to write queries that utilize the `@timestamp` field without encountering type mismatch errors, ensuring accurate time-based operations and comparisons across the combined dataset. + +**index: events_date** + +``` +{ + "mappings": { + "properties": { + "@timestamp": { "type": "date" }, + "client_ip": { "type": "ip" }, + "event_duration": { "type": "long" }, + "message": { "type": "keyword" } + } + } +} +``` + +**index: events_date_nanos** + +``` +{ + "mappings": { + "properties": { + "@timestamp": { "type": "date_nanos" }, + "client_ip": { "type": "ip" }, + "event_duration": { "type": "long" }, + "message": { "type": "keyword" } + } + } +} +``` + +```esql +FROM events_date* +| EVAL date = @timestamp::date +| KEEP @timestamp, date, client_ip, event_duration, message +| SORT date +``` + +| @timestamp:date_nanos | date:date | client_ip:ip | event_duration:long | message:keyword | +|--------------------------| --- |--------------|---------| --- | +| null |1969-10-23T13:33:34.937Z| 172.21.0.5 | 1232382 |Disconnected| +| 2023-10-23T12:15:03.360Z |2023-10-23T12:15:03.360Z| 172.21.2.162 | 3450233 |Connected to 10.1.0.3| +| 2023-10-23T12:15:03.360103847Z|2023-10-23T12:15:03.360Z| 172.22.2.162 | 3450233 |Connected to 10.1.0.3| +| 2023-10-23T12:27:28.948Z |2023-10-23T12:27:28.948Z| 172.22.2.113 | 2764889 |Connected to 10.1.0.2| +| 2023-10-23T12:27:28.948Z |2023-10-23T12:27:28.948Z| 172.21.2.113 | 2764889 |Connected to 10.1.0.2| +| 2023-10-23T13:33:34.937193Z |2023-10-23T13:33:34.937Z| 172.22.0.5 | 1232382 |Disconnected| +| null |2263-10-23T13:51:54.732Z| 172.21.3.15 | 725448 |Connection error| ## Index metadata [esql-multi-index-index-metadata] diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index b2142037de5b1..458fffaa7f2b7 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -58,7 +58,6 @@ import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode.SYNC; import static org.elasticsearch.xpack.esql.tools.ProfileParser.parseProfile; @@ -715,9 +714,7 @@ public void testSuggestedCast() throws IOException { Map results = entityAsMap(resp); List columns = (List) results.get("columns"); DataType suggestedCast = DataType.suggestedCast(Set.of(listOfTypes.get(i), listOfTypes.get(j))); - if (IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled() - && isMillisOrNanos(listOfTypes.get(i)) - && isMillisOrNanos(listOfTypes.get(j))) { + if (isMillisOrNanos(listOfTypes.get(i)) && isMillisOrNanos(listOfTypes.get(j))) { // datetime and date_nanos are casted to date_nanos implicitly assertThat(columns, equalTo(List.of(Map.ofEntries(Map.entry("name", "my_field"), Map.entry("type", "date_nanos"))))); } else { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 606bd434ec9b8..e7b6ce320fc55 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -353,7 +353,7 @@ public enum Cap { /** * Support implicit casting for union typed fields that are mixed with date and date_nanos type. */ - IMPLICIT_CASTING_DATE_AND_DATE_NANOS(Build.current().isSnapshot()), + IMPLICIT_CASTING_DATE_AND_DATE_NANOS, /** * Support for named or positional parameters in EsqlQueryRequest. diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 50a061ab0d6ef..17491dedf495f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -149,7 +149,6 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE; -import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; @@ -192,7 +191,7 @@ public class Analyzer extends ParameterizedRuleExecutor( "Resolution", @@ -1975,42 +1974,32 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { */ private static class DateMillisToNanosInEsRelation extends Rule { - private final boolean isSnapshot; - - DateMillisToNanosInEsRelation(boolean isSnapshot) { - this.isSnapshot = isSnapshot; - } - @Override public LogicalPlan apply(LogicalPlan plan) { - if (isSnapshot) { - return plan.transformUp(EsRelation.class, relation -> { - if (relation.indexMode() == IndexMode.LOOKUP) { - return relation; + return plan.transformUp(EsRelation.class, relation -> { + if (relation.indexMode() == IndexMode.LOOKUP) { + return relation; + } + return relation.transformExpressionsUp(FieldAttribute.class, f -> { + if (f.field() instanceof InvalidMappedField imf && imf.types().stream().allMatch(DataType::isDate)) { + HashMap typeResolutions = new HashMap<>(); + var convert = new ToDateNanos(f.source(), f); + imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions)); + var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions); + return new FieldAttribute( + f.source(), + f.parentName(), + f.qualifier(), + f.name(), + resolvedField, + f.nullable(), + f.id(), + f.synthetic() + ); } - return relation.transformExpressionsUp(FieldAttribute.class, f -> { - if (f.field() instanceof InvalidMappedField imf && imf.types().stream().allMatch(DataType::isDate)) { - HashMap typeResolutions = new HashMap<>(); - var convert = new ToDateNanos(f.source(), f); - imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions)); - var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions); - return new FieldAttribute( - f.source(), - f.parentName(), - f.qualifier(), - f.name(), - resolvedField, - f.nullable(), - f.id(), - f.synthetic() - ); - } - return f; - }); + return f; }); - } else { - return plan; - } + }); } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index 95a7204b5c71f..3e595869d8f88 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -4141,7 +4141,6 @@ public void testBucketWithIntervalInStringInGroupingReferencedInAggregation() { } public void testImplicitCastingForDateAndDateNanosFields() { - assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); Analyzer analyzer = AnalyzerTestUtils.analyzer(indexWithUnionTypedFields); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index d1bb7aeaa166a..6b46fcf0b9596 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -2384,7 +2384,6 @@ public void testMatchFunctionStatisWithNonPushableCondition() { } public void testToDateNanosPushDown() { - assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields)); var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long"); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java index 5683f57c37965..720cd77dd2c8a 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java @@ -11,7 +11,6 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.EsqlTestUtils; -import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.Verifier; @@ -326,7 +325,6 @@ OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRe } public void testToDateNanos() { - assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields)); var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long"); diff --git a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml index 4e714a0c8eff6..4017adc5bc325 100644 --- a/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml +++ b/x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml @@ -913,7 +913,7 @@ suggested_type: - method: POST path: /_query parameters: [] - capabilities: [suggested_cast, implicit_casting_date_and_date_nanos] + capabilities: [suggested_cast, implicit_casting_date_and_date_nanos, aggregate_metric_double_rendering] reason: "date and date_nanos should no longer produce suggested_cast column" - do: