[ES|QL] Date nanos implicit casting in union types option elastic#2 (elastic#127797)

* implicit casting for union typed fields mixed with datetime and date_nanos

Author: fang-xing-esql

docs/changelog/127797.yaml

@@ -0,0 +1,6 @@
+pr: 127797
+summary: "Date nanos implicit casting in union types option #2"
+area: ES|QL
+type: enhancement
+issues:
+ - 110009

x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java

@@ -65,7 +65,7 @@ public class DateUtils {
         .appendValue(MINUTE_OF_HOUR, 2)
         .appendLiteral(':')
         .appendValue(SECOND_OF_MINUTE, 2)
-        .appendFraction(NANO_OF_SECOND, 3, 9, true)
+        .appendFraction(NANO_OF_SECOND, 0, 9, true)
         .appendOffsetId()
         .toFormatter(Locale.ROOT);
 

x-pack/plugin/esql/qa/server/single-node/build.gradle

@@ -13,6 +13,7 @@ dependencies {
   javaRestTestImplementation project(xpackModule('esql:qa:testFixtures'))
   javaRestTestImplementation project(xpackModule('esql:qa:server'))
   javaRestTestImplementation project(xpackModule('esql:tools'))
+  javaRestTestImplementation project(xpackModule('esql'))
   yamlRestTestImplementation project(xpackModule('esql:qa:server'))
 
   javaRestTestImplementation('org.apache.arrow:arrow-vector:16.1.0')

x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java

@@ -55,6 +55,8 @@
 import static org.elasticsearch.test.ListMatcher.matchesList;
 import static org.elasticsearch.test.MapMatcher.assertMap;
 import static org.elasticsearch.test.MapMatcher.matchesMap;
+import static org.elasticsearch.xpack.esql.action.EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS;
+import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos;
 import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode.SYNC;
 import static org.elasticsearch.xpack.esql.tools.ProfileParser.parseProfile;
 import static org.elasticsearch.xpack.esql.tools.ProfileParser.readProfileFromResponse;
@@ -736,19 +738,26 @@ public void testSuggestedCast() throws IOException {
                 Map<String, Object> results = entityAsMap(resp);
                 List<?> columns = (List<?>) results.get("columns");
                 DataType suggestedCast = DataType.suggestedCast(Set.of(listOfTypes.get(i), listOfTypes.get(j)));
-                assertThat(
-                    columns,
-                    equalTo(
-                        List.of(
-                            Map.ofEntries(
-                                Map.entry("name", "my_field"),
-                                Map.entry("type", "unsupported"),
-                                Map.entry("original_types", List.of(listOfTypes.get(i).typeName(), listOfTypes.get(j).typeName())),
-                                Map.entry("suggested_cast", suggestedCast.typeName())
+                if (IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()
+                    && isMillisOrNanos(listOfTypes.get(i))
+                    && isMillisOrNanos(listOfTypes.get(j))) {
+                    // datetime and date_nanos are casted to date_nanos implicitly
+                    assertThat(columns, equalTo(List.of(Map.ofEntries(Map.entry("name", "my_field"), Map.entry("type", "date_nanos")))));
+                } else {
+                    assertThat(
+                        columns,
+                        equalTo(
+                            List.of(
+                                Map.ofEntries(
+                                    Map.entry("name", "my_field"),
+                                    Map.entry("type", "unsupported"),
+                                    Map.entry("original_types", List.of(listOfTypes.get(i).typeName(), listOfTypes.get(j).typeName())),
+                                    Map.entry("suggested_cast", suggestedCast.typeName())
+                                )
                             )
                         )
-                    )
-                );
+                    );
+                }
 
                 String castedQuery = String.format(
                     Locale.ROOT,

x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java

@@ -131,6 +131,7 @@ public class CsvTestsDataLoader {
     );
     private static final TestDataset AIRPORTS_WEB = new TestDataset("airports_web");
     private static final TestDataset DATE_NANOS = new TestDataset("date_nanos");
+    private static final TestDataset DATE_NANOS_UNION_TYPES = new TestDataset("date_nanos_union_types");
     private static final TestDataset COUNTRIES_BBOX = new TestDataset("countries_bbox");
     private static final TestDataset COUNTRIES_BBOX_WEB = new TestDataset("countries_bbox_web");
     private static final TestDataset AIRPORT_CITY_BOUNDARIES = new TestDataset("airport_city_boundaries");
@@ -201,6 +202,7 @@ public class CsvTestsDataLoader {
         Map.entry(MULTIVALUE_GEOMETRIES.indexName, MULTIVALUE_GEOMETRIES),
         Map.entry(MULTIVALUE_POINTS.indexName, MULTIVALUE_POINTS),
         Map.entry(DATE_NANOS.indexName, DATE_NANOS),
+        Map.entry(DATE_NANOS_UNION_TYPES.indexName, DATE_NANOS_UNION_TYPES),
         Map.entry(K8S.indexName, K8S),
         Map.entry(DISTANCES.indexName, DISTANCES),
         Map.entry(ADDRESSES.indexName, ADDRESSES),

x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_union_types.csv

@@ -0,0 +1,13 @@
+millis:date_nanos,nanos:date,num:long
+2023-10-23T13:55:01.543123456Z,2023-10-23T13:55:01.543Z,1698069301543123456
+2023-10-23T13:55:01.543123456Z,2023-10-23T13:55:01.543Z,1698069301543123456
+2023-10-23T13:53:55.832987654Z,2023-10-23T13:53:55.832Z,1698069235832987654
+2023-10-23T13:52:55.015787878Z,2023-10-23T13:52:55.015Z,1698069175015787878
+2023-10-23T13:51:54.732102837Z,2023-10-23T13:51:54.732Z,1698069114732102837
+2023-10-23T13:33:34.937193000Z,2023-10-23T13:33:34.937Z,1698068014937193000
+2023-10-23T12:27:28.948000000Z,2023-10-23T12:27:28.948Z,1698064048948000000
+2023-10-23T12:15:03.360103847Z,2023-10-23T12:15:03.360Z,1698063303360103847
+2023-10-23T12:15:03.360103847Z,2023-10-23T12:15:03.360Z,1698063303360103847
+1999-10-23T12:15:03.360103847Z,[2023-03-23T12:15:03.360Z, 2023-02-23T13:33:34.937Z, 2023-01-23T13:55:01.543Z], 0
+1999-10-22T12:15:03.360103847Z,[2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z], 0
+2023-10-23T12:15:03.360103847Z,1923-10-23T12:15:03.360Z,1698063303360103847

x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv

@@ -1,4 +1,4 @@
-birth_date:date_nanos     ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword
+birth_date:date     ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword
 1953-09-02T00:00:00Z,10001,Georgi     ,M,1986-06-26T00:00:00Z,2,2,2,2,Facello     ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19]
 1964-06-02T00:00:00Z,10002,Bezalel    ,F,1985-11-21T00:00:00Z,5,5,5,5,Simmel      ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17]
 1959-12-03T00:00:00Z,10003,Parto      ,M,1986-08-28T00:00:00Z,4,4,4,4,Bamford     ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82]

x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_union_types.json

@@ -0,0 +1,13 @@
+{
+    "properties": {
+        "millis": {
+            "type": "date_nanos"
+        },
+        "nanos": {
+            "type": "date"
+        },
+        "num": {
+            "type": "long"
+        }
+    }
+}