diff --git a/docs/changelog/123678.yaml b/docs/changelog/123678.yaml new file mode 100644 index 0000000000000..f32b0f966fb7c --- /dev/null +++ b/docs/changelog/123678.yaml @@ -0,0 +1,6 @@ +pr: 123678 +summary: Date nanos implicit casting +area: ES|QL +type: enhancement +issues: + - 110009 diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java index 20f7b400e9364..ae316460dd11c 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java @@ -65,7 +65,7 @@ public class DateUtils { .appendValue(MINUTE_OF_HOUR, 2) .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2) - .appendFraction(NANO_OF_SECOND, 3, 9, true) + .appendFraction(NANO_OF_SECOND, 0, 9, true) .appendOffsetId() .toFormatter(Locale.ROOT); diff --git a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java index d902f32c16c45..545b01f4f3066 100644 --- a/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java +++ b/x-pack/plugin/esql/qa/server/single-node/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/single_node/RestEsqlIT.java @@ -55,6 +55,7 @@ import static org.elasticsearch.test.ListMatcher.matchesList; import static org.elasticsearch.test.MapMatcher.assertMap; import static org.elasticsearch.test.MapMatcher.matchesMap; +import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; import static org.elasticsearch.xpack.esql.qa.rest.RestEsqlTestCase.Mode.SYNC; import static org.elasticsearch.xpack.esql.tools.ProfileParser.parseProfile; import static org.elasticsearch.xpack.esql.tools.ProfileParser.readProfileFromResponse; @@ -724,6 +725,10 @@ public void testSuggestedCast() throws IOException { for (int i = 0; i < listOfTypes.size(); i++) { for (int j = i + 1; j < listOfTypes.size(); j++) { + if (isMillisOrNanos(listOfTypes.get(i)) && isMillisOrNanos(listOfTypes.get(j))) { + // datetime and date_nanos are casted to date_nanos implicitly + continue; + } String query = String.format(Locale.ROOT, """ { "query": "FROM index-%s,index-%s | LIMIT 100 | KEEP my_field" diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java index 2abe77fe08c89..bdb47c0e50046 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/CsvTestsDataLoader.java @@ -125,6 +125,7 @@ public class CsvTestsDataLoader { ); private static final TestDataset AIRPORTS_WEB = new TestDataset("airports_web"); private static final TestDataset DATE_NANOS = new TestDataset("date_nanos"); + private static final TestDataset DATE_NANOS_UNION_TYPES = new TestDataset("date_nanos_union_types"); private static final TestDataset COUNTRIES_BBOX = new TestDataset("countries_bbox"); private static final TestDataset COUNTRIES_BBOX_WEB = new TestDataset("countries_bbox_web"); private static final TestDataset AIRPORT_CITY_BOUNDARIES = new TestDataset("airport_city_boundaries"); @@ -192,6 +193,7 @@ public class CsvTestsDataLoader { Map.entry(MULTIVALUE_GEOMETRIES.indexName, MULTIVALUE_GEOMETRIES), Map.entry(MULTIVALUE_POINTS.indexName, MULTIVALUE_POINTS), Map.entry(DATE_NANOS.indexName, DATE_NANOS), + Map.entry(DATE_NANOS_UNION_TYPES.indexName, DATE_NANOS_UNION_TYPES), Map.entry(K8S.indexName, K8S), Map.entry(DISTANCES.indexName, DISTANCES), Map.entry(ADDRESSES.indexName, ADDRESSES), diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_union_types.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_union_types.csv new file mode 100644 index 0000000000000..a212dc72ccf80 --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/date_nanos_union_types.csv @@ -0,0 +1,13 @@ +millis:date_nanos,nanos:date,num:long +2023-10-23T13:55:01.543123456Z,2023-10-23T13:55:01.543Z,1698069301543123456 +2023-10-23T13:55:01.543123456Z,2023-10-23T13:55:01.543Z,1698069301543123456 +2023-10-23T13:53:55.832987654Z,2023-10-23T13:53:55.832Z,1698069235832987654 +2023-10-23T13:52:55.015787878Z,2023-10-23T13:52:55.015Z,1698069175015787878 +2023-10-23T13:51:54.732102837Z,2023-10-23T13:51:54.732Z,1698069114732102837 +2023-10-23T13:33:34.937193000Z,2023-10-23T13:33:34.937Z,1698068014937193000 +2023-10-23T12:27:28.948000000Z,2023-10-23T12:27:28.948Z,1698064048948000000 +2023-10-23T12:15:03.360103847Z,2023-10-23T12:15:03.360Z,1698063303360103847 +2023-10-23T12:15:03.360103847Z,2023-10-23T12:15:03.360Z,1698063303360103847 +1999-10-23T12:15:03.360103847Z,[2023-03-23T12:15:03.360Z, 2023-02-23T13:33:34.937Z, 2023-01-23T13:55:01.543Z], 0 +1999-10-22T12:15:03.360103847Z,[2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z, 2023-03-23T12:15:03.360Z], 0 +2023-10-23T12:15:03.360103847Z,1923-10-23T12:15:03.360Z,1698063303360103847 diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv index ddbdb89476c4c..718bb3add9b1c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv @@ -1,4 +1,4 @@ -birth_date:date_nanos ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword +birth_date:date ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword 1953-09-02T00:00:00Z,10001,Georgi ,M,1986-06-26T00:00:00Z,2,2,2,2,Facello ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19] 1964-06-02T00:00:00Z,10002,Bezalel ,F,1985-11-21T00:00:00Z,5,5,5,5,Simmel ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17] 1959-12-03T00:00:00Z,10003,Parto ,M,1986-08-28T00:00:00Z,4,4,4,4,Bamford ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_union_types.json b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_union_types.json new file mode 100644 index 0000000000000..1017639bb5cfc --- /dev/null +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/mapping-date_nanos_union_types.json @@ -0,0 +1,13 @@ +{ + "properties": { + "millis": { + "type": "date_nanos" + }, + "nanos": { + "type": "date" + }, + "num": { + "type": "long" + } + } +} diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index aa08799943ade..ef1dd9bc56ba4 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1706,3 +1706,709 @@ id:integer | name:keyword | count:long 13 | lllll | 2 14 | mmmmm | 2 ; + +ImplicitCastingMultiTypedFieldsKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, y = hire_date::datetime, z = hire_date::date_nanos +| KEEP x, languages, hire_date, avg_worked_seconds, y, z +| SORT x, z +| LIMIT 2 +; + +x:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |y:datetime |z:date_nanos +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsDropKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, x = hire_date::datetime, y = hire_date +| DROP salary +| KEEP emp_no, languages, hire_date, avg_worked_seconds, x, y +| SORT emp_no, y +| limit 2 +; + +emp_no:long |languages:unsupported |hire_date:date_nanos |avg_worked_seconds:unsigned_long |x:datetime |y:date_nanos +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10001 |null |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsRenameKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +from employees, employees_incompatible +| EVAL emp_no = emp_no::long, avg_worked_seconds = avg_worked_seconds::unsigned_long, languages = languages::integer, x = hire_date::datetime +| RENAME emp_no as new_emp_no, languages as new_languages, hire_date as new_hire_date, avg_worked_seconds as new_avg_worked_seconds, x as y +| KEEP new_emp_no, new_languages, new_hire_date, new_avg_worked_seconds, y +| SORT new_emp_no, new_hire_date +| limit 2 +; + +new_emp_no:long |new_languages:integer |new_hire_date:date_nanos |new_avg_worked_seconds:unsigned_long |y:datetime +10001 |2 |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z +10001 |2 |1986-06-26T00:00:00.000Z |268728049 |1986-06-26T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsDropEvalKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| DROP birth_date +| EVAL x = emp_no::long + 1, y = hire_date + 1 year, z = hire_date::datetime +| KEEP x, hire_date, y, z +| SORT x, hire_date +| limit 2 +; + +x:long |hire_date:date_nanos |y:date_nanos |z:date +10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date == "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterNotEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date != "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10014 | 1985-10-20T00:00:00.000Z +10014 | 1985-10-20T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterGreaterThanKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date > "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long | hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10020 | 1999-04-30T00:00:00.000Z +10020 | 1999-04-30T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterGreaterThanOrEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date >= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterLessThanKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date < "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10014 | 1985-10-20T00:00:00.000Z +10014 | 1985-10-20T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterLessThanOrEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long | hire_date:date_nanos +10014 | 1985-10-20T00:00:00.000Z +10014 | 1985-10-20T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterLessThanOrEqualNotEqualKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1993-08-03" AND hire_date != "1985-10-20" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long | hire_date:date_nanos +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +10019 | 1987-04-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeExcludeLowerExcludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date < "1995-01-27" AND hire_date > "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10045 | 1994-05-21T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeIncludeLowerIncludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1995-01-27" AND hire_date >= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeExcludeLowerIncludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date <= "1995-01-27" AND hire_date > "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10017 | 1995-01-27T00:00:00.000Z +10017 | 1995-01-27T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsEvalFilterRangeIncludeLowerExcludeUpperKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL x = emp_no::long + 1, y = hire_date + 1 day +| WHERE x > 10011 AND hire_date < "1995-01-27" AND hire_date >= "1993-08-03" AND languages::integer < 3 AND height::double > 1.2 +| KEEP x, hire_date +| SORT x, hire_date +| limit 4 +; + +x:long |hire_date:date_nanos +10018 | 1993-08-03T00:00:00.000Z +10018 | 1993-08-03T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +10045 | 1994-05-21T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsStatsByNumeric +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=max(hire_date), y = min(hire_date) BY languages = languages::integer +| SORT languages +; + +x:date_nanos | y:date_nanos | languages:integer +1999-04-30T00:00:00.000Z | 1985-02-18T00:00:00.000Z | 1 +1995-01-27T00:00:00.000Z | 1986-03-14T00:00:00.000Z | 2 +1996-11-05T00:00:00.000Z | 1985-02-24T00:00:00.000Z | 3 +1995-03-13T00:00:00.000Z | 1985-05-13T00:00:00.000Z | 4 +1994-04-09T00:00:00.000Z | 1985-11-19T00:00:00.000Z | 5 +1997-05-19T00:00:00.000Z | 1985-11-20T00:00:00.000Z | null +; + +ImplicitCastingMultiTypedFieldsStatsByNumericWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=max(hire_date) where hire_date < "1993-08-03", y = min(hire_date) where hire_date >= "1993-08-03" BY languages = languages::integer +| SORT languages +; + +x:date_nanos | y:date_nanos | languages:integer +1990-10-20T00:00:00.000Z | 1994-05-21T00:00:00.000Z | 1 +1991-06-26T00:00:00.000Z | 1993-08-03T00:00:00.000Z | 2 +1993-03-21T00:00:00.000Z | 1994-02-17T00:00:00.000Z | 3 +1993-02-14T00:00:00.000Z | 1995-03-13T00:00:00.000Z | 4 +1992-12-18T00:00:00.000Z | 1994-04-09T00:00:00.000Z | 5 +1991-10-22T00:00:00.000Z | 1995-03-20T00:00:00.000Z | null +; + +ImplicitCastingMultiTypedFieldsStatsByDateNanos +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=count(emp_no::long), y=avg(salary_change::double), z=max(height::double) BY hire_date +| Eval y = round(y, 1), z = round(z, 1) +| KEEP x, y, z, hire_date +| SORT hire_date +| LIMIT 5 +; + +x:long | y:double | z:double | hire_date:date_nanos +2 | null | 1.9 | 1985-02-18T00:00:00.000Z +2 | null | 2.0 | 1985-02-24T00:00:00.000Z +2 | 3.3 | 2.0 | 1985-05-13T00:00:00.000Z +2 | 0.2 | 1.8 | 1985-07-09T00:00:00.000Z +2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsStatsByDateNanosWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS x=count(emp_no::long) where hire_date > "1985-05-01", y=avg(salary_change::double) where hire_date > "1985-05-01", z=max(height::double) where hire_date > "1985-05-01" BY hire_date +| Eval y = round(y, 1), z = round(z, 1) +| KEEP x, y, z, hire_date +| SORT hire_date +| LIMIT 5 +; + +x:long | y:double | z:double | hire_date:date_nanos +0 | null | null | 1985-02-18T00:00:00.000Z +0 | null | null | 1985-02-24T00:00:00.000Z +2 | 3.3 | 2.0 | 1985-05-13T00:00:00.000Z +2 | 0.2 | 1.8 | 1985-07-09T00:00:00.000Z +2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z +; + +ImplicitCastingMultiTypedFieldsMvExpandKeepSort +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| MV_EXPAND nanos +| SORT nanos +| LIMIT 4 +; + +warning:Line 2:13: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:13: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds + +num:long | nanos:date_nanos | millis:date_nanos +0 | 2023-01-23T13:55:01.543Z | 1999-10-23T12:15:03.360103847Z +0 | 2023-01-23T13:55:01.543123456Z | 1999-10-23T12:15:03.360Z +0 | 2023-02-23T13:33:34.937Z | 1999-10-23T12:15:03.360103847Z +0 | 2023-02-23T13:33:34.937193Z | 1999-10-23T12:15:03.360Z +; + +ImplicitCastingMultiTypedMVFieldsEval +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| EVAL nanos = MV_MAX(nanos) +| SORT nanos, millis +| LIMIT 4 +; + +warning:Line 2:23: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:23: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +num:long | nanos:date_nanos | millis:date_nanos +0 | 2023-03-23T12:15:03.360Z | 1999-10-22T12:15:03.360103847Z +0 | 2023-03-23T12:15:03.360Z | 1999-10-23T12:15:03.360103847Z +0 | 2023-03-23T12:15:03.360103847Z | 1999-10-22T12:15:03.360Z +0 | 2023-03-23T12:15:03.360103847Z | 1999-10-23T12:15:03.360Z +; + +ImplicitCastingMultiTypedMVFieldsWhere +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| WHERE millis <= "2023-10-23T13:00:00" AND MV_MAX(nanos) > "2023-03-23T13:00:00" +| KEEP millis, nanos, num +| SORT millis +; + +warning:Line 2:52: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:52: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds +millis:date_nanos | nanos:date_nanos | num:long +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360Z | 2023-10-23T12:15:03.360103847Z | 1698063303360103847 +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 +2023-10-23T12:15:03.360103847Z | 2023-10-23T12:15:03.360Z | 1698063303360103847 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 +2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948Z | 1698064048948000000 +; + +ImplicitCastingMultiTypedMVFieldsStatsMaxMin +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| STATS max = MAX(millis), min = MIN(nanos) +; + +warning:Line 2:38: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:38: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds + +max:date_nanos | min:date_nanos +2023-10-23T13:55:01.543123456Z | 2023-01-23T13:55:01.543Z +; + +ImplicitCastingMultiTypedMVFieldsStatsValues +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM date_nanos, date_nanos_union_types +| STATS c = MV_COUNT(VALUES(nanos)) +; + +warning:Line 2:29: evaluation of [nanos] failed, treating result as null. Only first 20 failures recorded. +warning:Line 2:29: java.lang.IllegalArgumentException: milliSeconds [-1457696696640] are before the epoch in 1970 and cannot be converted to nanoseconds + +c:integer +19 +; + +ImplicitCastingMultiTypedDateDiff +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long +| WHERE emp_no <= 10003 +| EVAL diff_sec_1 = DATE_DIFF("seconds", TO_DATE_NANOS("1986-01-23T12:15:03.360103847Z"), hire_date) +| EVAL diff_sec_2 = DATE_DIFF("seconds", TO_DATETIME("1986-01-23T12:15:03.360103847Z"), hire_date) +| KEEP emp_no, hire_date, diff_sec_1, diff_sec_2 +| SORT emp_no, hire_date +; + +emp_no:long | hire_date:date_nanos | diff_sec_1:integer | diff_sec_2:integer +10001 | 1986-06-26T00:00:00.000Z | 13261496 | 13261496 +10001 | 1986-06-26T00:00:00.000Z | 13261496 | 13261496 +10002 | 1985-11-21T00:00:00.000Z | -5487303 | -5487303 +10002 | 1985-11-21T00:00:00.000Z | -5487303 | -5487303 +10003 | 1986-08-28T00:00:00.000Z | 18704696 | 18704696 +10003 | 1986-08-28T00:00:00.000Z | 18704696 | 18704696 +; + +ImplicitCastingMultiTypedDateFormat +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long +| WHERE emp_no <= 10003 +| EVAL a = DATE_FORMAT(hire_date), b = DATE_FORMAT("yyyy-MM-dd", hire_date), c = DATE_FORMAT("strict_date_optional_time_nanos", hire_date) +| KEEP emp_no, hire_date, a, b, c +| SORT emp_no, hire_date +; + +emp_no:long | hire_date:date_nanos | a:keyword | b:keyword | c:keyword +10001 | 1986-06-26T00:00:00.000Z | 1986-06-26T00:00:00.000Z | 1986-06-26 | 1986-06-26T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-06-26T00:00:00.000Z | 1986-06-26 | 1986-06-26T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-11-21T00:00:00.000Z | 1985-11-21 | 1985-11-21T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-11-21T00:00:00.000Z | 1985-11-21 | 1985-11-21T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-08-28T00:00:00.000Z | 1986-08-28 | 1986-08-28T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-08-28T00:00:00.000Z | 1986-08-28 | 1986-08-28T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTrunc +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL emp_no = emp_no::long +| WHERE emp_no <= 10003 +| EVAL yr = DATE_TRUNC(1 year, hire_date), mo = DATE_TRUNC(1 month, hire_date) +| SORT hire_date DESC +| KEEP emp_no, hire_date, yr, mo +; + +emp_no:long | hire_date:date_nanos | yr:date_nanos | mo:date_nanos +10003 | 1986-08-28T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-08-01T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-08-01T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-06-01T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-06-01T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncStatsBy +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(emp_no::long) BY yr = DATE_TRUNC(1 year, hire_date) +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncStatsByWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(emp_no::long) where hire_date > "1996-01-01" BY yr = DATE_TRUNC(1 year, hire_date) +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +0 | 1995-01-01T00:00:00.000Z +0 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncStatsByWithEval +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL yr = DATE_TRUNC(1 year, hire_date) +| STATS c = count(emp_no::long) BY yr +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedDateTruncStatsByWithEvalWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| EVAL yr = DATE_TRUNC(1 year, hire_date) +| STATS c = count(emp_no::long) where hire_date > "1991-01-01" BY yr +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosByYear +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*) BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +24 | 1990-01-01T00:00:00.000Z +26 | 1989-01-01T00:00:00.000Z +18 | 1988-01-01T00:00:00.000Z +30 | 1987-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z +22 | 1985-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosByYearWithFilter +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*) where hire_date > "1991-01-01" BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +0 | 1990-01-01T00:00:00.000Z +0 | 1989-01-01T00:00:00.000Z +0 | 1988-01-01T00:00:00.000Z +0 | 1987-01-01T00:00:00.000Z +0 | 1986-01-01T00:00:00.000Z +0 | 1985-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosByMonth +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*) BY mo = BUCKET(hire_date, 20, "1986-01-01", "1999-12-31") +| SORT mo DESC, c +; + +c:long | mo:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +24 | 1990-01-01T00:00:00.000Z +26 | 1989-01-01T00:00:00.000Z +18 | 1988-01-01T00:00:00.000Z +30 | 1987-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z +22 | 1985-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosInBothStatsAndBy +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*), b = BUCKET(hire_date, 1 year) + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | b:date_nanos | yr:date_nanos +2 | 2000-01-01T00:00:00.000Z | 1999-01-01T00:00:00.000Z +2 | 1998-01-01T00:00:00.000Z | 1997-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z | 1996-01-01T00:00:00.000Z +10 | 1996-01-01T00:00:00.000Z | 1995-01-01T00:00:00.000Z +8 | 1995-01-01T00:00:00.000Z | 1994-01-01T00:00:00.000Z +6 | 1994-01-01T00:00:00.000Z | 1993-01-01T00:00:00.000Z +16 | 1993-01-01T00:00:00.000Z | 1992-01-01T00:00:00.000Z +12 | 1992-01-01T00:00:00.000Z | 1991-01-01T00:00:00.000Z +24 | 1991-01-01T00:00:00.000Z | 1990-01-01T00:00:00.000Z +26 | 1990-01-01T00:00:00.000Z | 1989-01-01T00:00:00.000Z +18 | 1989-01-01T00:00:00.000Z | 1988-01-01T00:00:00.000Z +30 | 1988-01-01T00:00:00.000Z | 1987-01-01T00:00:00.000Z +22 | 1987-01-01T00:00:00.000Z | 1986-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z +; + +ImplicitCastingMultiTypedBucketDateNanosInBothStatsAndByWithAlias +required_capability: date_nanos_type +required_capability: implicit_casting_date_and_date_nanos + +FROM employees, employees_incompatible +| STATS c = count(*), b = yr + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | b:date_nanos | yr:date_nanos +2 | 2000-01-01T00:00:00.000Z | 1999-01-01T00:00:00.000Z +2 | 1998-01-01T00:00:00.000Z | 1997-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z | 1996-01-01T00:00:00.000Z +10 | 1996-01-01T00:00:00.000Z | 1995-01-01T00:00:00.000Z +8 | 1995-01-01T00:00:00.000Z | 1994-01-01T00:00:00.000Z +6 | 1994-01-01T00:00:00.000Z | 1993-01-01T00:00:00.000Z +16 | 1993-01-01T00:00:00.000Z | 1992-01-01T00:00:00.000Z +12 | 1992-01-01T00:00:00.000Z | 1991-01-01T00:00:00.000Z +24 | 1991-01-01T00:00:00.000Z | 1990-01-01T00:00:00.000Z +26 | 1990-01-01T00:00:00.000Z | 1989-01-01T00:00:00.000Z +18 | 1989-01-01T00:00:00.000Z | 1988-01-01T00:00:00.000Z +30 | 1988-01-01T00:00:00.000Z | 1987-01-01T00:00:00.000Z +22 | 1987-01-01T00:00:00.000Z | 1986-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index fe22f1737a7f9..f6069678b1233 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -307,6 +307,11 @@ public enum Cap { */ STRING_LITERAL_AUTO_CASTING_TO_DATETIME_ADD_SUB, + /** + * Support implicit casting for union typed fields that are mixed with date and date_nanos type. + */ + IMPLICIT_CASTING_DATE_AND_DATE_NANOS(Build.current().isSnapshot()), + /** * Support for named or positional parameters in EsqlQueryRequest. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 3e2ffa706b441..65a15bd26110f 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -9,6 +9,7 @@ import org.elasticsearch.common.logging.HeaderWarning; import org.elasticsearch.common.logging.LoggerMessageFormat; +import org.elasticsearch.common.util.Maps; import org.elasticsearch.compute.data.Block; import org.elasticsearch.core.Strings; import org.elasticsearch.index.IndexMode; @@ -17,6 +18,7 @@ import org.elasticsearch.xpack.esql.Column; import org.elasticsearch.xpack.esql.EsqlIllegalArgumentException; import org.elasticsearch.xpack.esql.VerificationException; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.AnalyzerRules.ParameterizedAnalyzerRule; import org.elasticsearch.xpack.esql.common.Failure; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; @@ -60,6 +62,8 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FoldablesConvertFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; @@ -90,6 +94,7 @@ import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; +import org.elasticsearch.xpack.esql.plan.logical.UnaryPlan; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; import org.elasticsearch.xpack.esql.plan.logical.inference.Completion; import org.elasticsearch.xpack.esql.plan.logical.inference.InferencePlan; @@ -143,9 +148,12 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; +import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount; @@ -158,7 +166,7 @@ public class Analyzer extends ParameterizedRuleExecutor NO_FIELDS = List.of( - new ReferenceAttribute(Source.EMPTY, NO_FIELDS_NAME, DataType.NULL, Nullability.TRUE, null, true) + new ReferenceAttribute(Source.EMPTY, NO_FIELDS_NAME, NULL, Nullability.TRUE, null, true) ); private static final List> RULES = List.of( @@ -175,7 +183,9 @@ public class Analyzer extends ParameterizedRuleExecutor("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new AddImplicitForkLimit(), new UnionTypesCleanup()) ); @@ -678,7 +688,7 @@ private LogicalPlan resolveLookup(Lookup l, List childrenOutput) { */ boolean dataTypesOk = joinedAttribute.dataType().equals(attr.dataType()); if (false == dataTypesOk) { - dataTypesOk = joinedAttribute.dataType() == DataType.NULL || attr.dataType() == DataType.NULL; + dataTypesOk = joinedAttribute.dataType() == NULL || attr.dataType() == NULL; } if (false == dataTypesOk) { dataTypesOk = joinedAttribute.dataType().equals(KEYWORD) && attr.dataType().equals(TEXT); @@ -1344,7 +1354,7 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { *
  • date_trunc("1 minute", dateField)
    • * * If the inputs to Coalesce are mixed numeric types, cast the rest of the numeric field or value to the first numeric data type if - * applicable. For example, implicit casting converts: + * applicable, the same applies to Case, Greatest, Least. For example, implicit casting converts: *
      *
    • Coalesce(Long, Int) to Coalesce(Long, Long)
      • *
    • Coalesce(null, Long, Int) to Coalesce(null, Long, Long)
      • @@ -1356,6 +1366,7 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { private static class ImplicitCasting extends ParameterizedRule { @Override public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { + // do implicit casting for function arguments return plan.transformExpressionsUp( org.elasticsearch.xpack.esql.core.expression.function.Function.class, e -> ImplicitCasting.cast(e, context.functionRegistry().snapshotRegistry()) @@ -1386,7 +1397,7 @@ private static Expression processScalarOrGroupingFunction( } List newChildren = new ArrayList<>(args.size()); boolean childrenChanged = false; - DataType targetDataType = DataType.NULL; + DataType targetDataType = NULL; Expression arg; DataType targetNumericType = null; boolean castNumericArgs = true; @@ -1399,7 +1410,7 @@ private static Expression processScalarOrGroupingFunction( if (i < targetDataTypes.size()) { targetDataType = targetDataTypes.get(i); } - if (targetDataType != DataType.NULL && targetDataType != DataType.UNSUPPORTED) { + if (targetDataType != NULL && targetDataType != UNSUPPORTED) { Expression e = castStringLiteral(arg, targetDataType); if (e != arg) { childrenChanged = true; @@ -1432,7 +1443,7 @@ private static Expression processBinaryOperator(BinaryOperator o) { } List newChildren = new ArrayList<>(2); boolean childrenChanged = false; - DataType targetDataType = DataType.NULL; + DataType targetDataType = NULL; Expression from = Literal.NULL; if (left.dataType() == KEYWORD && left.foldable() && (left instanceof EsqlScalarFunction == false)) { @@ -1787,4 +1798,184 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { return newOutput.size() == output.size() ? plan : new Project(Source.EMPTY, plan, newOutput); } } + + /** + * Cast union typed fields that are mixed of date and date_nanos types into date_nanos. + */ + private static class ImplicitCastingForUnionTypedFields extends ParameterizedRule { + @Override + public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { + if (EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled() == false) { + return plan; + } + // This rule should be applied after ResolveUnionTypes, so that the InvalidMappedFields with explicit casting are converted into + // MultiTypeEsField, and don't get double cast here. + Map invalidMappedFieldCasted = new HashMap<>(); + LogicalPlan transformedPlan = plan.transformUp(LogicalPlan.class, p -> { + // exclude LookupJoin for now, as it doesn't support date_nanos as join key yet + if (p instanceof UnaryPlan == false) { + return p; + } + Set invalidMappedFields = invalidMappedFieldsInLogicalPlan(p); + if (invalidMappedFields.isEmpty() == false) { + // If we are at a plan node that has invalid mapped fields, we need to either add an EVAL, or if that has been done + // we should instead replace with the already cast field + Map newAliases = Maps.newHashMapWithExpectedSize(invalidMappedFields.size()); + Map existingAliases = Maps.newHashMapWithExpectedSize(invalidMappedFields.size()); + for (FieldAttribute fa : invalidMappedFields) { + if (invalidMappedFieldCasted.containsKey(fa)) { + // There is already an eval plan created for the implicit cast field, just reference to it + Alias alias = invalidMappedFieldCasted.get(fa); + existingAliases.put(fa, alias); + } else { + // Create a new alias and later on add a new EVAL with this new aliases for implicit casting + DataType targetType = commonDataType(fa); + if (targetType != null) { + Expression conversionFunction = castInvalidMappedField(targetType, fa); + Alias alias = new Alias(fa.source(), fa.name(), conversionFunction); + newAliases.put(fa, alias); + invalidMappedFieldCasted.put(fa, alias); + } + } + } + // If there are new aliases created, create a new eval child with new aliases for the current plan。 + // How many children does a LogicalPlan have? Only deal with UnaryPlan and LookupJoin for now. + if (newAliases.isEmpty() == false) { // create a new eval child plan + UnaryPlan u = (UnaryPlan) p; // this must be a unary plan, as it is checked at the beginning of plan loop + Eval eval = new Eval(u.source(), u.child(), newAliases.values().stream().toList()); + p = u.replaceChild(eval); + // TODO Lookup join does not work on date_nanos field yet, joining on a date_nanos field does not find a match. + // And lookup up join is a special case as a lookup join has two children, after date_nanos is supported as a join + // key, the transformation needs to take it into account. + } + // If there are new or existing aliases identified, combine them into one map + Map allAliases = Maps.newHashMapWithExpectedSize(invalidMappedFields.size()); + allAliases.putAll(newAliases); + allAliases.putAll(existingAliases); + if (allAliases.isEmpty() == false) { // there is already eval plan for that union typed field, reference to the aliases + p = p.transformExpressionsOnly(FieldAttribute.class, fa -> { + Alias alias = allAliases.get(fa); + return alias != null ? alias.toAttribute() : fa; + }); + // MvExpand and Stats have ReferenceAttribute referencing the FieldAttribute in the same plan. + // The ReferenceAttribute need to be updated to point to the casting expression. + if (p instanceof MvExpand mvExpand) { + p = transformMvExpand(mvExpand); + } else if (p instanceof Aggregate aggregate) { + p = transformAggregate(aggregate); + } + } + } + return p; + }); + transformedPlan = castInvalidMappedFieldInFinalOutput(transformedPlan); + return transformedPlan; + } + + /** + * Find a common data type that the union typed field can cast to, only date and date_nanos types are supported. + * This method can be extended to support implicit casting for the other data types. + */ + private static DataType commonDataType(FieldAttribute unionTypedField) { + DataType targetType = null; + if (unionTypedField.field() instanceof InvalidMappedField imf) { + for (DataType type : imf.types()) { + if (isMillisOrNanos(type) == false) { // if there is field that is no date or date_nanos, don't do implicit casting + return null; + } + if (targetType == null) { // initialize the target type to the first type + targetType = type; + } else if (targetType == DATE_NANOS || type == DATE_NANOS) { + targetType = DATE_NANOS; + } + } + } + return targetType; + } + + /** + * Do implicit casting for date and date_nanos only. + */ + private static Expression castInvalidMappedField(DataType targetType, FieldAttribute fa) { + Source source = fa.source(); + return switch (targetType) { + case DATETIME -> new ToDatetime(source, fa); // in case we decided to use DATE as a common type instead of DATE_NANOS + case DATE_NANOS -> new ToDateNanos(source, fa); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + }; + } + + /** + * Return all the FieldAttribute that contain InvalidMappedField in the current plan. + */ + private static Set invalidMappedFieldsInLogicalPlan(LogicalPlan plan) { + Set fas = new HashSet<>(); + // Invalid mapped fields are legal at EsRelation level, as long as they are not used elsewhere. In the final output, if they + // have not been dropped, implicit cast will be added for them, so that we can return not null values, the implicit casting is + // deferred to when the fields are used or returned. + if (plan instanceof EsRelation == false) { + plan.forEachExpression(FieldAttribute.class, fa -> { + if (fa.field() instanceof InvalidMappedField) { + fas.add(fa); + } + }); + } + return fas; + } + + /** + * Cast the InvalidMappedFields in the final output of the query, this is needed when these fields are not referenced in the query + * explicitly, so there is no chance to cast them to a common type earlier, an example of such query is from index*. + */ + private static LogicalPlan castInvalidMappedFieldInFinalOutput(LogicalPlan logicalPlan) { + // Check the output of the query, if the top level plan is resolved, check if there is InvalidMappedField in its output, + // if so add a project with eval, so that a not null value can be returned for a union typed field + if (logicalPlan.resolved()) { + List output = logicalPlan.output(); + Map newAliases = Maps.newHashMapWithExpectedSize(output.size()); + output.forEach(a -> { + if (a instanceof FieldAttribute fa && fa.field() instanceof InvalidMappedField) { + DataType targetType = commonDataType(fa); + if (targetType != null) { + Expression conversionFunction = castInvalidMappedField(targetType, fa); + Alias alias = new Alias(fa.source(), fa.name(), conversionFunction); + newAliases.put(fa, alias); + } + } + }); + if (newAliases.isEmpty() == false) { // add an Eval for the union typed fields left that are not cast implicitly yet + if (logicalPlan instanceof EsRelation esr) { + // EsRelation does not have a child, we should not see row here, add a eval on top of it + logicalPlan = new Eval(esr.source(), esr, newAliases.values().stream().toList()); + } else if (logicalPlan instanceof UnaryPlan unary) { + // Add an Eval as the child of this plan + Eval eval = new Eval(unary.source(), unary.child(), newAliases.values().stream().toList()); + logicalPlan = unary.replaceChild(eval); + } + // TODO LookupJoin is a binary plan, it does not create a new field, ideally adding an Eval on top of it should be fine, + // however because the output of a LookupJoin does not include InvalidMappedFields even the LHS output has + // InvalidMappedFields, it is a bug need to be addressed + } + } + return logicalPlan; + } + + private static MvExpand transformMvExpand(MvExpand mvExpand) { + NamedExpression target = mvExpand.target(); + return new MvExpand(mvExpand.source(), mvExpand.child(), target, target.toAttribute()); + } + + private static Aggregate transformAggregate(Aggregate aggregate) { + List aggregates = aggregate.aggregates(); + List groupings = aggregate.groupings(); + List aggregatesWithNewRefs = new ArrayList<>(aggregates.size()); + for (int i = 0; i < aggregates.size() - groupings.size(); i++) { + aggregatesWithNewRefs.add(aggregates.get(i)); + } + for (Expression e : groupings) { // Add groupings + aggregatesWithNewRefs.add(Expressions.attribute(e)); + } + return aggregate.with(groupings, aggregatesWithNewRefs); + } + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java index 30ea63d0e473d..ec4abd49f1864 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java @@ -14,6 +14,7 @@ import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FoldContext; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.expression.function.scalar.ScalarFunction; import org.elasticsearch.xpack.esql.core.expression.predicate.operator.comparison.BinaryComparison; import org.elasticsearch.xpack.esql.core.querydsl.query.Query; @@ -279,7 +280,13 @@ private RangeQuery translate(TranslatorHandler handler) { } } logger.trace("Building range query with format string [{}]", format); - return new RangeQuery(source(), handler.nameOf(value), l, includeLower(), u, includeUpper(), format, zoneId); + // This is a similar check as in EsqlBinaryComparison + // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found + TypedAttribute attribute = LucenePushdownPredicates.checkIsPushableAttribute(value); + String name = handler.nameOf(attribute); + String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(attribute); + name = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : name; + return new RangeQuery(source(), name, l, includeLower(), u, includeUpper(), format, zoneId); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java index 542c696fd3521..6e39130f43a9e 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/EsqlBinaryComparison.java @@ -378,6 +378,9 @@ public Expression singleValueField() { private Query translate(TranslatorHandler handler) { TypedAttribute attribute = LucenePushdownPredicates.checkIsPushableAttribute(left()); String name = handler.nameOf(attribute); + // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found + String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(attribute); + name = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : name; Object value = valueOf(FoldContext.small() /* TODO remove me */, right()); String format = null; boolean isDateLiteralComparison = false; @@ -452,7 +455,10 @@ private Query translate(TranslatorHandler handler) { return new RangeQuery(source(), name, null, false, value, true, format, zoneId); } if (this instanceof Equals || this instanceof NotEquals) { - name = LucenePushdownPredicates.pushableAttributeName(attribute); + // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found + name = fieldNameFromMultiTypeEsField != null + ? fieldNameFromMultiTypeEsField + : LucenePushdownPredicates.pushableAttributeName(attribute); Query query; if (isDateLiteralComparison) { diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java index 7843f8a6cfe04..a51a7f1a51154 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/optimizer/rules/physical/local/LucenePushdownPredicates.java @@ -7,14 +7,22 @@ package org.elasticsearch.xpack.esql.optimizer.rules.physical.local; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.MetadataAttribute; import org.elasticsearch.xpack.esql.core.expression.TypedAttribute; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.Check; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.stats.SearchStats; +import java.util.Map; +import java.util.function.Predicate; + +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; + /** * When deciding if a filter or topN can be pushed down to Lucene, we need to check a few things on the field. * Exactly what is checked depends on the type of field and the query. For example, we have the following possible combinations: @@ -96,6 +104,46 @@ static String pushableAttributeName(TypedAttribute attribute) { : attribute.name(); } + /** + * Extract the real field name from a MultiTypeEsField, limit to MultiTypeEsField that has date_nanos type only. + * + * For example, the name of a MultiTypeEsField can be $$myfield$converted_to$date_nanos, and the real field name extract from the + * MultiTypeEsField is myfield, this method return myfield given a MultiTypeEsField. + * + * If the real field name is found, and the original field data types contain only date and date_nanos types, return the real field + * name, so that the real field name will be used to check for eligibility of being pushed down, and the real field name will be used + * in the push down query, instead of the name of the MultiTypeEsField, which should not match any field in an index. + * + * This method can be extended to support the other data types in the future if there is a need. + */ + static String extractFieldNameFromMultiTypeEsField(TypedAttribute attribute) { + if (EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled() + && attribute instanceof FieldAttribute fa + && fa.field() instanceof MultiTypeEsField multiTypeEsField + && fa.dataType() == DATE_NANOS + && // limit to casting to date_nanos only + mixedDateAndDateNanosOnly(multiTypeEsField, DataType::isMillisOrNanos) // limit to mixed date and date_nanos only + ) { + return fa.fieldName(); + } + return null; + } + + /** + * Check if the original field types in a MultiTypeEsField satisfy the required data types defined in the predicate. + */ + private static boolean mixedDateAndDateNanosOnly(MultiTypeEsField multiTypeEsField, Predicate predicate) { + Map indexToConversionExpressions = multiTypeEsField.getIndexToConversionExpressions(); + for (Map.Entry entry : indexToConversionExpressions.entrySet()) { + Expression conversionFunction = entry.getValue(); + if (conversionFunction instanceof AbstractConvertFunction abstractConvertFunction + && predicate.test(abstractConvertFunction.field().dataType()) == false) { + return false; + } + } + return true; + } + /** * The default implementation of this has no access to SearchStats, so it can only make decisions based on the FieldAttribute itself. * In particular, it assumes TEXT fields have no exact subfields (underlying keyword field), @@ -138,10 +186,14 @@ public boolean hasExactSubfield(FieldAttribute attr) { @Override public boolean isIndexedAndHasDocValues(FieldAttribute attr) { + // If this is a MultiTypeEsField cast to date_nanos, make it eligible for being pushed down by checking the real + // field name against SearchStats + String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(attr); + String name = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : attr.name(); // We still consider the value of isAggregatable here, because some fields like ScriptFieldTypes are always aggregatable // But this could hide issues with fields that are not indexed but are aggregatable // This is the original behaviour for ES|QL, but is it correct? - return attr.field().isAggregatable() || stats.isIndexed(attr.name()) && stats.hasDocValues(attr.name()); + return attr.field().isAggregatable() || stats.isIndexed(name) && stats.hasDocValues(name); } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java index f65811fc26526..52941018ffae8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; @@ -90,7 +91,7 @@ public String telemetryLabel() { @Override public boolean expressionsResolved() { - return target.resolved(); + return target.resolved() && target.dataType() != DataType.UNSUPPORTED; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java index 4b7af5bf49de8..e06d2531cc4a9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/TranslatorHandler.java @@ -46,7 +46,10 @@ private static Query wrapFunctionQuery(Expression field, Query query) { } if (field instanceof FieldAttribute fa) { fa = fa.getExactInfo().hasExact() ? fa.exactAttribute() : fa; - return new SingleValueQuery(query, fa.name(), false); + // Extract the real field name from MultiTypeEsField, and use it in the push down query if it is found + String fieldNameFromMultiTypeEsField = LucenePushdownPredicates.extractFieldNameFromMultiTypeEsField(fa); + String fieldName = fieldNameFromMultiTypeEsField != null ? fieldNameFromMultiTypeEsField : fa.name(); + return new SingleValueQuery(query, fieldName, false); } if (field instanceof MetadataAttribute) { return query; // MetadataAttributes are always single valued diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java index 704a0395e97b4..87b5adbfde8b4 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTestUtils.java @@ -11,6 +11,8 @@ import org.elasticsearch.inference.TaskType; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.InvalidMappedField; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.index.EsIndex; @@ -24,8 +26,10 @@ import org.elasticsearch.xpack.esql.session.Configuration; import java.util.ArrayList; +import java.util.LinkedHashMap; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.function.Predicate; import java.util.function.Supplier; @@ -220,4 +224,25 @@ public static E randomValueOtherThanTest(Predicate exclude, Supplier s } } } + + public static IndexResolution indexWithDateDateNanosUnionType() { + // this method is shared by AnalyzerTest, QueryTranslatorTests and LocalPhysicalPlanOptimizerTests + String dateDateNanos = "date_and_date_nanos"; // mixed date and date_nanos + String dateDateNanosLong = "date_and_date_nanos_and_long"; // mixed date, date_nanos and long + LinkedHashMap> typesToIndices1 = new LinkedHashMap<>(); + typesToIndices1.put("date", Set.of("index1", "index2")); + typesToIndices1.put("date_nanos", Set.of("index3")); + LinkedHashMap> typesToIndices2 = new LinkedHashMap<>(); + typesToIndices2.put("date", Set.of("index1")); + typesToIndices2.put("date_nanos", Set.of("index2")); + typesToIndices2.put("long", Set.of("index3")); + EsField dateDateNanosField = new InvalidMappedField(dateDateNanos, typesToIndices1); + EsField dateDateNanosLongField = new InvalidMappedField(dateDateNanosLong, typesToIndices2); + EsIndex index = new EsIndex( + "test*", + Map.of(dateDateNanos, dateDateNanosField, dateDateNanosLong, dateDateNanosLongField), + Map.of("index1", IndexMode.STANDARD, "index2", IndexMode.STANDARD, "index3", IndexMode.STANDARD) + ); + return IndexResolution.valid(index); + } } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java index f3afa24969f33..44e6831d7f68f 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/AnalyzerTests.java @@ -35,6 +35,7 @@ import org.elasticsearch.xpack.esql.core.expression.UnresolvedAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.type.PotentiallyUnmappedKeywordEsField; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; @@ -73,6 +74,7 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.OrderBy; +import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Row; import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; import org.elasticsearch.xpack.esql.plan.logical.UnresolvedRelation; @@ -110,6 +112,7 @@ import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzerDefaultMapping; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.defaultEnrichResolution; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.randomValueOtherThanTest; import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.tsdbIndexResolution; @@ -3937,6 +3940,77 @@ public void testBucketWithIntervalInStringInGroupingReferencedInAggregation() { assertEquals(oneYear, literal); } + public void testImplicitCastingForDateAndDateNanosFields() { + assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); + IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); + Analyzer analyzer = AnalyzerTestUtils.analyzer(indexWithUnionTypedFields); + + // validate if a union typed field is cast to a type explicitly, implicit casting won't be applied again + LogicalPlan plan = analyze(""" + FROM tests + | Eval x = date_and_date_nanos::datetime, y = date_and_date_nanos, z = date_and_date_nanos::date_nanos + """, analyzer); + + Project project = as(plan, Project.class); + List projections = project.projections(); + assertEquals(5, projections.size()); + // long is not casted to date_nanos + UnsupportedAttribute ua = as(projections.get(0), UnsupportedAttribute.class); + assertEquals("date_and_date_nanos_and_long", ua.name()); + assertEquals(DataType.UNSUPPORTED, ua.dataType()); + // implicit casting + ReferenceAttribute ra = as(projections.get(1), ReferenceAttribute.class); + assertEquals("date_and_date_nanos", ra.name()); + assertEquals(DataType.DATE_NANOS, ra.dataType()); + // explicit casting + ra = as(projections.get(2), ReferenceAttribute.class); + assertEquals("x", ra.name()); + assertEquals(DataType.DATETIME, ra.dataType()); + // implicit casting + ra = as(projections.get(3), ReferenceAttribute.class); + assertEquals("y", ra.name()); + assertEquals(DataType.DATE_NANOS, ra.dataType()); + // explicit casting + ra = as(projections.get(4), ReferenceAttribute.class); + assertEquals("z", ra.name()); + assertEquals(DataType.DATE_NANOS, ra.dataType()); + Limit limit = as(project.child(), Limit.class); + // original Eval coded in the query + Eval eval = as(limit.child(), Eval.class); + List aliases = eval.fields(); + assertEquals(3, aliases.size()); + // explicit casting + Alias a = aliases.get(0); + assertEquals("x", a.name()); + assertEquals(DataType.DATETIME, a.dataType()); + assertTrue(isMultiTypeEsField(a.child())); + // implicit casting + a = aliases.get(1); + assertEquals("y", a.name()); + assertEquals(DataType.DATE_NANOS, a.dataType()); + assertTrue(a.child() instanceof ReferenceAttribute); + // explicit casting + a = aliases.get(2); + assertEquals("z", a.name()); + assertEquals(DataType.DATE_NANOS, a.dataType()); + assertTrue(isMultiTypeEsField(a.child())); + // a new eval added for implicit casting + eval = as(eval.child(), Eval.class); + aliases = eval.fields(); + assertEquals(1, aliases.size()); + a = aliases.get(0); + assertEquals("date_and_date_nanos", a.name()); + assertEquals(DataType.DATE_NANOS, a.dataType()); + assertTrue(isMultiTypeEsField(a.child())); + + EsRelation esRelation = as(eval.child(), EsRelation.class); + assertEquals("test*", esRelation.indexPattern()); + } + + private boolean isMultiTypeEsField(Expression e) { + return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; + } + @Override protected IndexAnalyzers createDefaultIndexAnalyzers() { return super.createDefaultIndexAnalyzers(); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java index e540649e8c602..550e31a31402b 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java @@ -39,19 +39,24 @@ import org.elasticsearch.xpack.esql.analysis.EnrichResolution; import org.elasticsearch.xpack.esql.analysis.Verifier; import org.elasticsearch.xpack.esql.core.expression.Alias; +import org.elasticsearch.xpack.esql.core.expression.Expression; import org.elasticsearch.xpack.esql.core.expression.Expressions; import org.elasticsearch.xpack.esql.core.expression.FieldAttribute; import org.elasticsearch.xpack.esql.core.expression.Literal; +import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.Source; import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.core.type.EsField; +import org.elasticsearch.xpack.esql.core.type.MultiTypeEsField; import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.enrich.ResolvedEnrichPolicy; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; +import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; import org.elasticsearch.xpack.esql.expression.function.fulltext.Match; import org.elasticsearch.xpack.esql.expression.predicate.logical.Or; import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThan; +import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.GreaterThanOrEqual; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.index.IndexResolution; import org.elasticsearch.xpack.esql.optimizer.rules.logical.ExtractAggregateCommonFilter; @@ -111,7 +116,9 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.unboundLogicalOptimizerContext; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; import static org.elasticsearch.xpack.esql.core.querydsl.query.Query.unscore; +import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.plan.physical.EsStatsQueryExec.StatsType; import static org.hamcrest.Matchers.contains; import static org.hamcrest.Matchers.equalTo; @@ -141,6 +148,7 @@ public class LocalPhysicalPlanOptimizerTests extends MapperServiceTestCase { public static final String MATCH_FUNCTION_QUERY = "from test | where match(%s, %s)"; private TestPlannerOptimizer plannerOptimizer; + private TestPlannerOptimizer plannerOptimizerDateDateNanosUnionTypes; private Analyzer timeSeriesAnalyzer; private final Configuration config; private final SearchStats IS_SV_STATS = new TestSearchStats() { @@ -213,6 +221,13 @@ private Analyzer makeAnalyzer(String mappingFileName) { return makeAnalyzer(mappingFileName, new EnrichResolution()); } + private Analyzer makeAnalyzer(IndexResolution indexResolution) { + return new Analyzer( + new AnalyzerContext(config, new EsqlFunctionRegistry(), indexResolution, new EnrichResolution(), emptyInferenceResolution()), + new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L)) + ); + } + /** * Expects * LimitExec[1000[INTEGER]] @@ -1856,6 +1871,56 @@ public void testPushDownFieldExtractToTimeSeriesSource() { assertTrue(timeSeriesSource.attrs().stream().noneMatch(EsQueryExec::isSourceAttribute)); } + public void testToDateNanosPushDown() { + assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); + IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); + plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields)); + var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long"); + String query = """ + from test* + | where date_and_date_nanos < "2025-01-01" and date_and_date_nanos_and_long::date_nanos >= "2024-01-01\""""; + var plan = plannerOptimizerDateDateNanosUnionTypes.plan(query, stats); + + // date_and_date_nanos should be pushed down to EsQueryExec, date_and_date_nanos_and_long should not be pushed down + var project = as(plan, ProjectExec.class); + List projections = project.projections(); + assertEquals(2, projections.size()); + UnsupportedAttribute ua = as(projections.get(0), UnsupportedAttribute.class); // mixed date, date_nanos and long are not auto-casted + assertEquals("date_and_date_nanos_and_long", ua.fieldName()); + Alias alias = as(projections.get(1), Alias.class); + assertEquals(DATE_NANOS, alias.dataType()); + FieldAttribute ra = as(alias.child(), FieldAttribute.class); + assertEquals("date_and_date_nanos", ra.fieldName()); + assertTrue(isMultiTypeEsField(ra)); // mixed date and date_nanos are auto-casted + var limit = as(project.child(), LimitExec.class); + var exchange = as(limit.child(), ExchangeExec.class); + project = as(exchange.child(), ProjectExec.class); + var fieldExtract = as(project.child(), FieldExtractExec.class); + limit = as(fieldExtract.child(), LimitExec.class); + // date_and_date_nanos_and_long::date_nanos >= "2024-01-01" is not pushed down + var filter = as(limit.child(), FilterExec.class); + GreaterThanOrEqual gt = as(filter.condition(), GreaterThanOrEqual.class); + FieldAttribute fa = as(gt.left(), FieldAttribute.class); + assertTrue(isMultiTypeEsField(fa)); + assertEquals("date_and_date_nanos_and_long", fa.fieldName()); + fieldExtract = as(filter.child(), FieldExtractExec.class); // extract date_and_date_nanos_and_long + var esQuery = as(fieldExtract.child(), EsQueryExec.class); + var source = ((SingleValueQuery.Builder) esQuery.query()).source(); + var expected = wrapWithSingleQuery( + query, + unscore( + rangeQuery("date_and_date_nanos").lt("2025-01-01T00:00:00.000Z").timeZone("Z").format("strict_date_optional_time_nanos") + ), + "date_and_date_nanos", + source + ); // date_and_date_nanos is pushed down + assertThat(expected.toString(), is(esQuery.query().toString())); + } + + private boolean isMultiTypeEsField(Expression e) { + return e instanceof FieldAttribute fa && fa.field() instanceof MultiTypeEsField; + } + private QueryBuilder wrapWithSingleQuery(String query, QueryBuilder inner, String fieldName, Source source) { return FilterTests.singleValueQuery(query, inner, fieldName, source); } diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java index 02b108dcf6adb..ff47227cd25f8 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java @@ -11,6 +11,7 @@ import org.elasticsearch.license.XPackLicenseState; import org.elasticsearch.test.ESTestCase; import org.elasticsearch.xpack.esql.EsqlTestUtils; +import org.elasticsearch.xpack.esql.action.EsqlCapabilities; import org.elasticsearch.xpack.esql.analysis.Analyzer; import org.elasticsearch.xpack.esql.analysis.AnalyzerContext; import org.elasticsearch.xpack.esql.analysis.Verifier; @@ -20,6 +21,7 @@ import org.elasticsearch.xpack.esql.optimizer.TestPlannerOptimizer; import org.elasticsearch.xpack.esql.plan.physical.EsQueryExec; import org.elasticsearch.xpack.esql.plan.physical.PhysicalPlan; +import org.elasticsearch.xpack.esql.stats.SearchStats; import org.elasticsearch.xpack.esql.telemetry.Metrics; import org.hamcrest.Matcher; import org.junit.BeforeClass; @@ -31,6 +33,7 @@ import static org.elasticsearch.xpack.esql.EsqlTestUtils.emptyPolicyResolution; import static org.elasticsearch.xpack.esql.EsqlTestUtils.loadMapping; import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning; +import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.indexWithDateDateNanosUnionType; import static org.hamcrest.Matchers.containsString; import static org.hamcrest.Matchers.matchesRegex; @@ -41,6 +44,8 @@ public class QueryTranslatorTests extends ESTestCase { private static TestPlannerOptimizer plannerOptimizerIPs; + private static TestPlannerOptimizer plannerOptimizerDateDateNanosUnionTypes; + private static Analyzer makeAnalyzer(String mappingFileName) { var mapping = loadMapping(mappingFileName); EsIndex test = new EsIndex("test", mapping, Map.of("test", IndexMode.STANDARD)); @@ -58,6 +63,19 @@ private static Analyzer makeAnalyzer(String mappingFileName) { ); } + public static Analyzer makeAnalyzer(IndexResolution indexResolution) { + return new Analyzer( + new AnalyzerContext( + EsqlTestUtils.TEST_CFG, + new EsqlFunctionRegistry(), + indexResolution, + emptyPolicyResolution(), + emptyInferenceResolution() + ), + new Verifier(new Metrics(new EsqlFunctionRegistry()), new XPackLicenseState(() -> 0L)) + ); + } + @BeforeClass public static void init() { plannerOptimizer = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer("mapping-all-types.json")); @@ -83,6 +101,13 @@ public void assertQueryTranslationIPs(String query, Matcher translationM assertThat(translatedQuery, translationMatcher); } + private void assertQueryTranslationDateDateNanosUnionTypes(String query, SearchStats stats, Matcher translationMatcher) { + PhysicalPlan optimized = plannerOptimizerDateDateNanosUnionTypes.plan(query, stats); + EsQueryExec eqe = (EsQueryExec) optimized.collectLeaves().get(0); + final String translatedQuery = eqe.query().toString().replaceAll("\\s+", ""); + assertThat(translatedQuery, translationMatcher); + } + public void testBinaryComparisons() { assertQueryTranslation(""" FROM test | WHERE 10 < integer""", containsString(""" @@ -299,4 +324,117 @@ OR CIDR_MATCH(ip0, "fe80::cae2:65ff:fece:feb9") OR host == "beta\"""", matchesRe esql_single_value":\\{"field":"ip1".*"terms":\\{"ip1":\\["127.0.0.3/32","127.0.0.2".*""" + """ esql_single_value":\\{"field":"ip0".*"terms":\\{"ip0":\\["127.0.0.1","128.0.0.1","fe80::cae2:65ff:fece:feb9".*""")); } + + public void testToDateNanos() { + assumeTrue("requires snapshot", EsqlCapabilities.Cap.IMPLICIT_CASTING_DATE_AND_DATE_NANOS.isEnabled()); + IndexResolution indexWithUnionTypedFields = indexWithDateDateNanosUnionType(); + plannerOptimizerDateDateNanosUnionTypes = new TestPlannerOptimizer(EsqlTestUtils.TEST_CFG, makeAnalyzer(indexWithUnionTypedFields)); + var stats = EsqlTestUtils.statsForExistingField("date_and_date_nanos", "date_and_date_nanos_and_long"); + + // == term + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos == "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"term":{"date_and_date_nanos":{"value":"2025-01-01T00:00:00.000Z","boost":0.0}}}""")); + + // != term + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos != "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"bool":{"must_not":[{"term":{"date_and_date_nanos":{"value":"2025-01-01T00:00:00.000Z","boost":0.0}}}],\ + "boost":0.0}}""")); + + // > range + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos > "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gt":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // >= range + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos >= "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gte":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // < range + assertQueryTranslationDateDateNanosUnionTypes( + """ + FROM test* | WHERE date_and_date_nanos < "2025-01-01" and date_and_date_nanos_and_long::date_nanos > "2025-01-01\"""", + stats, + containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"lt":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""") + ); + + // <= range + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos <= "2025-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"lte":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // <= and >= + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos <= "2025-01-01" and date_and_date_nanos > "2020-01-01\"""", stats, containsString(""" + "esql_single_value":{"field":"date_and_date_nanos",\ + "next":{"range":{"date_and_date_nanos":{"gt":"2020-01-01T00:00:00.000Z","lte":"2025-01-01T00:00:00.000Z","time_zone":"Z",\ + "format":"strict_date_optional_time_nanos","boost":0.0}}}""")); + + // >= or < + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos >= "2025-01-01" or date_and_date_nanos < "2020-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"gte":"2025-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"lt":"2020-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""")); + + // > or = + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos > "2025-01-01" or date_and_date_nanos == "2020-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"gt":"2025-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":"2020-01-01T00:00:00.000Z",\ + "boost":0.0.*""")); + + // < or != + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos < "2020-01-01" or date_and_date_nanos != "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"range":\\{"date_and_date_nanos":\\{"lt":"2020-01-01T00:00:00.000Z",\ + "time_zone":"Z","format":"strict_date_optional_time_nanos","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + + // == or == + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos == "2020-01-01" or date_and_date_nanos == "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":"2020-01-01T00:00:00.000Z",\ + "boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":"2025-01-01T00:00:00.000Z",\ + "boost":0.0.*""")); + + // != or != + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos != "2020-01-01" or date_and_date_nanos != "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2020-01-01T00:00:00.000Z","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + + // = or != + assertQueryTranslationDateDateNanosUnionTypes(""" + FROM test* | WHERE date_and_date_nanos == "2020-01-01" or date_and_date_nanos != "2025-01-01\"""", stats, matchesRegex(""" + .*bool.*should.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2020-01-01T00:00:00.000Z","boost":0.0.*""" + """ + esql_single_value":\\{"field":"date_and_date_nanos".*"must_not".*"term":\\{"date_and_date_nanos":\\{"value":\ + "2025-01-01T00:00:00.000Z","boost":0.0.*""")); + } }