[ES|QL] Take date date_nanos implicit casting out of snapshot

docs/changelog/133369.yaml

@@ -0,0 +1,5 @@
+pr: 133369
+summary: Enable `date` `date_nanos` implicit casting
+area: ES|QL
+type: enhancement
+issues: []

docs/reference/query-languages/esql/esql-multi-index.md

@@ -135,6 +135,60 @@ FROM events_*
 | 2023-10-23T12:27:28.948Z | 172.21.2.113 | 2764889 | Connected to 10.1.0.2 |
 | 2023-10-23T12:15:03.360Z | 172.21.2.162 | 3450233 | Connected to 10.1.0.3 |
 
+### Date and date_nanos union type [esql-multi-index-date-date-nanos-union]
+
+When the type of an {{esql}} field is a *union* of `date` and `date_nanos` across different indices, {{esql}} automatically casts all values to the `date_nanos` type during query execution. This implicit casting ensures that all values are handled with nanosecond precision, regardless of their original type. As a result, users can write queries against such fields without needing to perform explicit type conversions, and the query engine will seamlessly align the types for consistent and precise results.
+
+`date_nanos` fields offer higher precision but have a narrower valid value range compared to `date` fields, which limits their representable dates roughly from 1970 to 2262. This is because dates are stored as a long representing nanoseconds since the epoch. When a field is mapped as both `date` and `date_nanos` across different indices, {{esql}} defaults to the more precise date_nanos type. This behavior ensures that no precision is lost when querying multiple indices with differing date field types. For dates that fall outside the valid range of `date_nanos` in fields that are mapped to both `date` and `date_nanos` across different indices, {{esql}} returns null by default. However, users can explicitly cast these fields to the `date` type to obtain a valid value, with precision limited to milliseconds.
+
+For example, if the `@timestamp` field is mapped as `date` in one index and `date_nanos` in another, {{esql}} will automatically treat all `@timestamp` values as `date_nanos` during query execution. This allows users to write queries that utilize the `@timestamp` field without encountering type mismatch errors, ensuring accurate time-based operations and comparisons across the combined dataset.
+
+**index: events_date**
+
+```
+{
+  "mappings": {
+    "properties": {
+      "@timestamp":     { "type": "date" },
+      "client_ip":      { "type": "ip" },
+      "event_duration": { "type": "long" },
+      "message":        { "type": "keyword" }
+    }
+  }
+}
+```
+
+**index: events_date_nanos**
+
+```
+{
+  "mappings": {
+    "properties": {
+      "@timestamp":     { "type": "date_nanos" },
+      "client_ip":      { "type": "ip" },
+      "event_duration": { "type": "long" },
+      "message":        { "type": "keyword" }
+    }
+  }
+}
+```
+
+```esql
+FROM events_date*
+| EVAL date = @timestamp::date
+| KEEP @timestamp, date, client_ip, event_duration, message
+| SORT date
+```
+
+| @timestamp:date_nanos | date:date   | client_ip:ip | event_duration:long | message:keyword |
+|--------------------------| --- |--------------|---------| --- |
+| null                          |1969-10-23T13:33:34.937Z| 172.21.0.5   | 1232382 |Disconnected|
+| 2023-10-23T12:15:03.360Z      |2023-10-23T12:15:03.360Z| 172.21.2.162 | 3450233 |Connected to 10.1.0.3|
+| 2023-10-23T12:15:03.360103847Z|2023-10-23T12:15:03.360Z| 172.22.2.162 | 3450233 |Connected to 10.1.0.3|
+| 2023-10-23T12:27:28.948Z      |2023-10-23T12:27:28.948Z| 172.22.2.113 | 2764889 |Connected to 10.1.0.2|
+| 2023-10-23T12:27:28.948Z      |2023-10-23T12:27:28.948Z| 172.21.2.113 | 2764889 |Connected to 10.1.0.2|
+| 2023-10-23T13:33:34.937193Z   |2023-10-23T13:33:34.937Z| 172.22.0.5   | 1232382 |Disconnected|
+| null                          |2263-10-23T13:51:54.732Z| 172.21.3.15  | 725448  |Connection error|
 
 ## Index metadata [esql-multi-index-index-metadata]
 

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java

@@ -58,7 +58,6 @@
 import static org.elasticsearch.test.ListMatcher.matchesList;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS;
 import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos;
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode.SYNC;
 import static org.elasticsearch.xpack.esql.tools.ProfileParser.parseProfile;
@@ -712,9 +711,7 @@ public void testSuggestedCast() throws IOException {
                 Map<String, Object> results = entityAsMap(resp);
                 List<?> columns = (List<?>) results.get("columns");
                 DataType suggestedCast = DataType.suggestedCast(Set.of(listOfTypes.get(i), listOfTypes.get(j)));
-                if (IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()
-                    && isMillisOrNanos(listOfTypes.get(i))
-                    && isMillisOrNanos(listOfTypes.get(j))) {
+                if (isMillisOrNanos(listOfTypes.get(i)) && isMillisOrNanos(listOfTypes.get(j))) {
                     // datetime and date_nanos are casted to date_nanos implicitly
                     assertThat(columns, equalTo(List.of(Map.ofEntries(Map.entry("name", "my_field"), Map.entry("type", "date_nanos")))));
                 } else {

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java

@@ -353,7 +353,7 @@ public enum Cap {
         /**
          * Support implicit casting for union typed fields that are mixed with date and date_nanos type.
          */
-        IMPLICIT_CASTING_DATE_AND_DATE_NANOS(Build.current().isSnapshot()),
+        IMPLICIT_CASTING_DATE_AND_DATE_NANOS,
 
         /**
          * Support for named or positional parameters in EsqlQueryRequest.

x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java

@@ -149,7 +149,6 @@
 import static java.util.Collections.emptyList;
 import static java.util.Collections.singletonList;
 import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE;
-import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS;
 import static org.elasticsearch.xpack.esql.core.type.DataType.AGGREGATE_METRIC_DOUBLE;
 import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN;
 import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
@@ -192,7 +191,7 @@ public class Analyzer extends ParameterizedRuleExecutor<LogicalPlan, AnalyzerCon
             new ResolveLookupTables(),
             new ResolveFunctions(),
             new ResolveInference(),
-            new DateMillisToNanosInEsRelation(IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled())
+            new DateMillisToNanosInEsRelation()
         ),
         new Batch<>(
             "Resolution",
@@ -1975,42 +1974,32 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) {
      */
     private static class DateMillisToNanosInEsRelation extends Rule<LogicalPlan, LogicalPlan> {
 
-        private final boolean isSnapshot;
-
-        DateMillisToNanosInEsRelation(boolean isSnapshot) {
-            this.isSnapshot = isSnapshot;
-        }
-
         @Override
         public LogicalPlan apply(LogicalPlan plan) {
-            if (isSnapshot) {
-                return plan.transformUp(EsRelation.class, relation -> {
-                    if (relation.indexMode() == IndexMode.LOOKUP) {
-                        return relation;
+            return plan.transformUp(EsRelation.class, relation -> {
+                if (relation.indexMode() == IndexMode.LOOKUP) {
+                    return relation;
+                }
+                return relation.transformExpressionsUp(FieldAttribute.class, f -> {
+                    if (f.field() instanceof InvalidMappedField imf && imf.types().stream().allMatch(DataType::isDate)) {
+                        HashMap<ResolveUnionTypes.TypeResolutionKey, Expression> typeResolutions = new HashMap<>();
+                        var convert = new ToDateNanos(f.source(), f);
+                        imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions));
+                        var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions);
+                        return new FieldAttribute(
+                            f.source(),
+                            f.parentName(),
+                            f.qualifier(),
+                            f.name(),
+                            resolvedField,
+                            f.nullable(),
+                            f.id(),
+                            f.synthetic()
+                        );
                     }
-                    return relation.transformExpressionsUp(FieldAttribute.class, f -> {
-                        if (f.field() instanceof InvalidMappedField imf && imf.types().stream().allMatch(DataType::isDate)) {
-                            HashMap<ResolveUnionTypes.TypeResolutionKey, Expression> typeResolutions = new HashMap<>();
-                            var convert = new ToDateNanos(f.source(), f);
-                            imf.types().forEach(type -> typeResolutions(f, convert, type, imf, typeResolutions));
-                            var resolvedField = ResolveUnionTypes.resolvedMultiTypeEsField(f, typeResolutions);
-                            return new FieldAttribute(
-                                f.source(),
-                                f.parentName(),
-                                f.qualifier(),
-                                f.name(),
-                                resolvedField,
-                                f.nullable(),
-                                f.id(),
-                                f.synthetic()
-                            );
-                        }
-                        return f;
-                    });
+                    return f;
                 });
-            } else {
-                return plan;
-            }
+            });
         }
     }
 

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java

@@ -4142,7 +4142,6 @@ public void testBucketWithIntervalInStringInGroupingReferencedInAggregation() {
     }
 
     public void testImplicitCastingForDateAndDateNanosFields() {
-        assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled());
         IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType();
         Analyzer analyzer = AnalyzerTestUtils.analyzer(indexWithUnionTypedFields);
 

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java

@@ -2327,7 +2327,6 @@ public void testMatchFunctionStatisWithNonPushableCondition() {
     }
 
     public void testToDateNanosPushDown() {
-        assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled());
         IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType();
         plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields));
         var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long");

x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java

@@ -11,7 +11,6 @@
 import org.elasticsearch.license.XPackLicenseState;
 import org.elasticsearch.test.ESTestCase;
 import org.elasticsearch.xpack.esql.EsqlTestUtils;
-import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
 import org.elasticsearch.xpack.esql.analysis.Analyzer;
 import org.elasticsearch.xpack.esql.analysis.AnalyzerContext;
 import org.elasticsearch.xpack.esql.analysis.Verifier;
@@ -326,7 +325,6 @@ OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRe
     }
 
     public void testToDateNanos() {
-        assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled());
         IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType();
         plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields));
         var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long");

x-pack/plugin/src/yamlRestTest/resources/rest-api-spec/test/esql/160_union_types.yml

@@ -913,7 +913,7 @@ suggested_type:
         - method: POST
           path: /_query
           parameters: []
-          capabilities: [suggested_cast, implicit_casting_date_and_date_nanos]
+          capabilities: [suggested_cast, implicit_casting_date_and_date_nanos, aggregate_metric_double_rendering]
       reason: "date and date_nanos should no longer produce suggested_cast column"
 
   - do: