diff --git a/docs/changelog/123678.yaml b/docs/changelog/123678.yaml new file mode 100644 index 0000000000000..f32b0f966fb7c --- /dev/null +++ b/docs/changelog/123678.yaml @@ -0,0 +1,6 @@ +pr: 123678 +summary: Date nanos implicit casting +area: ES|QL +type: enhancement +issues: + - 110009 diff --git a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java index 20f7b400e9364..ae316460dd11c 100644 --- a/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java +++ b/x-pack/plugin/esql-core/src/main/java/org/elasticsearch/xpack/esql/core/util/DateUtils.java @@ -65,7 +65,7 @@ public class DateUtils { .appendValue(MINUTE_OF_HOUR, 2) .appendLiteral(':') .appendValue(SECOND_OF_MINUTE, 2) - .appendFraction(NANO_OF_SECOND, 3, 9, true) + .appendFraction(NANO_OF_SECOND, 0, 9, true) .appendOffsetId() .toFormatter(Locale.ROOT); diff --git a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java index 4cf8997f77be1..8d063dda416d6 100644 --- a/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java +++ b/x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/FieldExtractorTestCase.java @@ -959,6 +959,9 @@ public void testIntegerDocValuesConflict() throws IOException { * In an ideal world we'd promote the {@code integer} to an {@code long} and just go. */ public void testLongIntegerConflict() throws IOException { + if (EsqlCapabilities.Cap.IMPLICIT_CASTING_UNION_TYPED_NUMERIC_AND_DATE.isEnabled()) { + return; + } assumeOriginalTypesReported(); longTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no"); index("test1", """ @@ -1002,6 +1005,9 @@ public void testLongIntegerConflict() throws IOException { * In an ideal world we'd promote the {@code short} to an {@code integer} and just go. */ public void testIntegerShortConflict() throws IOException { + if (EsqlCapabilities.Cap.IMPLICIT_CASTING_UNION_TYPED_NUMERIC_AND_DATE.isEnabled()) { + return; + } assumeOriginalTypesReported(); intTest().sourceMode(SourceMode.DEFAULT).createIndex("test1", "emp_no"); index("test1", """ diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv index ddbdb89476c4c..718bb3add9b1c 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/data/employees_incompatible.csv @@ -1,4 +1,4 @@ -birth_date:date_nanos ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword +birth_date:date ,emp_no:long,first_name:text,gender:text,hire_date:date_nanos,languages:byte,languages.long:long,languages.short:short,languages.byte:byte,last_name:text,salary:long,height:float,height.double:double,height.scaled_float:scaled_float,height.half_float:half_float,still_hired:keyword,avg_worked_seconds:unsigned_long,job_positions:text,is_rehired:keyword,salary_change:float,salary_change.int:integer,salary_change.long:long,salary_change.keyword:keyword 1953-09-02T00:00:00Z,10001,Georgi ,M,1986-06-26T00:00:00Z,2,2,2,2,Facello ,57305,2.03,2.03,2.03,2.03,true ,268728049,[Senior Python Developer,Accountant],[false,true],[1.19],[1],[1],[1.19] 1964-06-02T00:00:00Z,10002,Bezalel ,F,1985-11-21T00:00:00Z,5,5,5,5,Simmel ,56371,2.08,2.08,2.08,2.08,true ,328922887,[Senior Team Lead],[false,false],[-7.23,11.17],[-7,11],[-7,11],[-7.23,11.17] 1959-12-03T00:00:00Z,10003,Parto ,M,1986-08-28T00:00:00Z,4,4,4,4,Bamford ,61805,1.83,1.83,1.83,1.83,false,200296405,[],[],[14.68,12.82],[14,12],[14,12],[14.68,12.82] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec index 8b19bc589fcff..0456f62941481 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/union_types.csv-spec @@ -1639,3 +1639,486 @@ id:integer | name:keyword | count:long 13 | lllll | 2 14 | mmmmm | 2 ; + +MultiTypedFieldsEvalKeepSort +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| Eval x = emp_no +| KEEP x, languages, hire_date, avg_worked_seconds +| SORT x, hire_date +| limit 2 +; + +x:long |languages:integer |hire_date:date_nanos |avg_worked_seconds:unsigned_long +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +; + +MultiTypedFieldsEvalMultipleTimes +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE birth_date > "1990-01-01" +| EVAL x = emp_no + 1, y = hire_date + 1 year, z = hire_date::datetime +| SORT z +| KEEP emp_no, x, hire_date, y, z +; + +emp_no:long |x:long |hire_date:date_nanos |y:date_nanos |z:date +10001 |10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +; + +MultiTypedFieldsDropKeepSort +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| DROP salary +| KEEP emp_no, languages, hire_date, avg_worked_seconds +| SORT emp_no, hire_date +| limit 2 +; + +emp_no:long |languages:integer |hire_date:date_nanos |avg_worked_seconds:unsigned_long +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +; + +MultiTypedFieldsRenameKeepSort +required_capability: implicit_casting_union_typed_numeric_and_date + +from employees, employees_incompatible +| RENAME emp_no as new_emp_no, languages as new_languages, hire_date as new_hire_date, avg_worked_seconds as new_avg_worked_seconds +| KEEP new_emp_no, new_languages, new_hire_date, new_avg_worked_seconds +| SORT new_emp_no, new_hire_date +| limit 2 +; + +new_emp_no:long |new_languages:integer |new_hire_date:date_nanos |new_avg_worked_seconds:unsigned_long +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +10001 |2 |1986-06-26T00:00:00.000Z |268728049 +; + +MultiTypedFieldsDropEvalKeepSort +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| DROP birth_date +| EVAL x = emp_no + 1, y = hire_date + 1 year, z = hire_date::datetime +| KEEP emp_no, x, hire_date, y, z +| SORT emp_no, hire_date +| limit 2 +; + +emp_no:long |x:long |hire_date:date_nanos |y:date_nanos |z:date +10001 |10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +10001 |10002 |1986-06-26T00:00:00.000Z |1987-06-26T00:00:00.000Z |1986-06-26T00:00:00.000Z +; + +MultiTypedFieldsEvalFilterKeepSort +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| EVAL x = emp_no + 1, y = hire_date + 1 day +| WHERE emp_no > 10010 AND hire_date > "1992-01-01" AND languages < 3 AND height > 1.2 +| KEEP emp_no, hire_date +| SORT emp_no, hire_date +| limit 4 +; + +emp_no:long |hire_date:date_nanos +10016 | 1995-01-27T00:00:00.000Z +10016 | 1995-01-27T00:00:00.000Z +10017 | 1993-08-03T00:00:00.000Z +10017 | 1993-08-03T00:00:00.000Z +; + +MultiTypedFieldsStatsByNumeric +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS x=count(emp_no), y=max(hire_date), z=max(height) BY languages +| SORT languages +; + +x:long | y:date_nanos | z:double | languages:integer +30 | 1999-04-30T00:00:00.000Z | 2.06 | 1 +38 | 1995-01-27T00:00:00.000Z | 2.1 | 2 +34 | 1996-11-05T00:00:00.000Z | 2.1 | 3 +36 | 1995-03-13T00:00:00.000Z | 2.0 | 4 +42 | 1994-04-09T00:00:00.000Z | 2.1 | 5 +20 | 1997-05-19T00:00:00.000Z | 2.1 | null +; + +MultiTypedFieldsStatsByDateNanos +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS x=count(emp_no), y=avg(salary_change), z=max(height) BY hire_date +| Eval y = round(y, 1), z = round(z, 1) +| KEEP x, y, z, hire_date +| SORT hire_date +| LIMIT 5 +; + +x:long | y:double | z:double | hire_date:date_nanos +2 | null | 1.9 | 1985-02-18T00:00:00.000Z +2 | null | 2.0 | 1985-02-24T00:00:00.000Z +2 | 3.3 | 2.0 | 1985-05-13T00:00:00.000Z +2 | 0.2 | 1.8 | 1985-07-09T00:00:00.000Z +2 | 3.6 | 1.5 | 1985-09-17T00:00:00.000Z +; + +MultiTypedFieldsWhereMvExpandKeepSortNumeric +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no == 10003 +| MV_EXPAND salary_change +| KEEP emp_no, salary_change +| EVAL salary_change = round(salary_change, 2) +| SORT salary_change +; + +emp_no:long | salary_change:double +10003 | 12.82 +10003 | 12.82 +10003 | 14.68 +10003 | 14.68 +; + +MultiTypedFieldsMvExpandKeepSortNumeric +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| MV_EXPAND salary_change +| KEEP emp_no, salary_change +| EVAL salary_change = round(salary_change, 2) +| SORT emp_no, salary_change +| LIMIT 10 +; + +emp_no:long | salary_change:double +10001 | 1.19 +10001 | 1.19 +10002 | -7.23 +10002 | -7.23 +10002 | 11.17 +10002 | 11.17 +10003 | 12.82 +10003 | 12.82 +10003 | 14.68 +10003 | 14.68 +; + +MultiTypedFieldsEvalLookupJoinNumeric +required_capability: join_lookup_v12 +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| EVAL language_code = languages +| LOOKUP JOIN languages_lookup ON language_code +| WHERE emp_no >= 10091 AND emp_no < 10094 +| KEEP emp_no, language_code, language_name +| SORT emp_no +; + +emp_no:long | language_code:integer | language_name:keyword +10091 | 3 | Spanish +10091 | 3 | Spanish +10092 | 1 | English +10092 | 1 | English +10093 | 3 | Spanish +10093 | 3 | Spanish +; + +MultiTypedMVFields +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10002 +| EVAL x = ROUND(MV_MAX(salary_change),2) +| KEEP emp_no, x +| SORT emp_no +| LIMIT 4 +; + +emp_no:long | x:double +10001 | 1.19 +10001 | 1.19 +10002 | 11.17 +10002 | 11.17 +; + +MultiTypedMVFieldsWhere +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10002 AND ROUND(MV_MAX(salary_change),2) > 10 +| KEEP emp_no +; + +emp_no:long +10002 +10002 +; + +MultiTypedMVFieldsStatsMaxMin +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10002 +| STATS max = MAX(salary_change), min = MIN(salary_change) +| EVAL x = round(max, 2), y = round(min, 2) +| KEEP x, y +; + +x:double |y:double +11.17 |-7.23 +; + +MultiTypedMVFieldsStatsValues +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10002 +| STATS c = MV_COUNT(VALUES(salary_change)) +; + +c:integer +6 +; + +MultiTypedDateDiff +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10003 +| EVAL diff_sec_1 = DATE_DIFF("seconds", TO_DATE_NANOS("1986-01-23T12:15:03.360103847Z"), hire_date) +| EVAL diff_sec_2 = DATE_DIFF("seconds", TO_DATETIME("1986-01-23T12:15:03.360103847Z"), hire_date) +| KEEP emp_no, hire_date, diff_sec_1, diff_sec_2 +| SORT emp_no, hire_date +; + +emp_no:long | hire_date:date_nanos | diff_sec_1:integer | diff_sec_2:integer +10001 | 1986-06-26T00:00:00.000Z | 13261496 | 13261496 +10001 | 1986-06-26T00:00:00.000Z | 13261496 | 13261496 +10002 | 1985-11-21T00:00:00.000Z | -5487303 | -5487303 +10002 | 1985-11-21T00:00:00.000Z | -5487303 | -5487303 +10003 | 1986-08-28T00:00:00.000Z | 18704696 | 18704696 +10003 | 1986-08-28T00:00:00.000Z | 18704696 | 18704696 +; + +MultiTypedDateFormat +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10003 +| EVAL a = DATE_FORMAT(hire_date), b = DATE_FORMAT("yyyy-MM-dd", hire_date), c = DATE_FORMAT("strict_date_optional_time_nanos", hire_date) +| KEEP emp_no, hire_date, a, b, c +| SORT emp_no, hire_date +; + +emp_no:long | hire_date:date_nanos | a:keyword | b:keyword | c:keyword +10001 | 1986-06-26T00:00:00.000Z | 1986-06-26T00:00:00.000Z | 1986-06-26 | 1986-06-26T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-06-26T00:00:00.000Z | 1986-06-26 | 1986-06-26T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-11-21T00:00:00.000Z | 1985-11-21 | 1985-11-21T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-11-21T00:00:00.000Z | 1985-11-21 | 1985-11-21T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-08-28T00:00:00.000Z | 1986-08-28 | 1986-08-28T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-08-28T00:00:00.000Z | 1986-08-28 | 1986-08-28T00:00:00.000Z +; + +MultiTypedDateTrunc +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| WHERE emp_no <= 10003 +| EVAL yr = DATE_TRUNC(1 year, hire_date), mo = DATE_TRUNC(1 month, hire_date) +| SORT hire_date DESC +| KEEP emp_no, hire_date, yr, mo +; + +emp_no:long | hire_date:date_nanos | yr:date_nanos | mo:date_nanos +10003 | 1986-08-28T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-08-01T00:00:00.000Z +10003 | 1986-08-28T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-08-01T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-06-01T00:00:00.000Z +10001 | 1986-06-26T00:00:00.000Z | 1986-01-01T00:00:00.000Z | 1986-06-01T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z +10002 | 1985-11-21T00:00:00.000Z | 1985-01-01T00:00:00.000Z | 1985-11-01T00:00:00.000Z +; + +MultiTypedDateTruncStatsBy +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(emp_no) BY yr = DATE_TRUNC(1 year, hire_date) +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +MultiTypedDateTruncStatsByWithEval +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| EVAL yr = DATE_TRUNC(1 year, hire_date) +| STATS c = count(emp_no) BY yr +| SORT yr DESC +| LIMIT 5 +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +; + +MultiTypedBucketDateNanosByYear +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(*) BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | yr:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +24 | 1990-01-01T00:00:00.000Z +26 | 1989-01-01T00:00:00.000Z +18 | 1988-01-01T00:00:00.000Z +30 | 1987-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z +22 | 1985-01-01T00:00:00.000Z +; + +MultiTypedBucketDateNanosByMonth +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(*) BY mo = BUCKET(hire_date, 20, "1986-01-01", "1999-12-31") +| SORT mo DESC, c +; + +c:long | mo:date_nanos +2 | 1999-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z +2 | 1996-01-01T00:00:00.000Z +10 | 1995-01-01T00:00:00.000Z +8 | 1994-01-01T00:00:00.000Z +6 | 1993-01-01T00:00:00.000Z +16 | 1992-01-01T00:00:00.000Z +12 | 1991-01-01T00:00:00.000Z +24 | 1990-01-01T00:00:00.000Z +26 | 1989-01-01T00:00:00.000Z +18 | 1988-01-01T00:00:00.000Z +30 | 1987-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z +22 | 1985-01-01T00:00:00.000Z +; + +MultiTypedBucketDateNanosInBothStatsAndBy +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(*), b = BUCKET(hire_date, 1 year) + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | b:date_nanos | yr:date_nanos +2 | 2000-01-01T00:00:00.000Z | 1999-01-01T00:00:00.000Z +2 | 1998-01-01T00:00:00.000Z | 1997-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z | 1996-01-01T00:00:00.000Z +10 | 1996-01-01T00:00:00.000Z | 1995-01-01T00:00:00.000Z +8 | 1995-01-01T00:00:00.000Z | 1994-01-01T00:00:00.000Z +6 | 1994-01-01T00:00:00.000Z | 1993-01-01T00:00:00.000Z +16 | 1993-01-01T00:00:00.000Z | 1992-01-01T00:00:00.000Z +12 | 1992-01-01T00:00:00.000Z | 1991-01-01T00:00:00.000Z +24 | 1991-01-01T00:00:00.000Z | 1990-01-01T00:00:00.000Z +26 | 1990-01-01T00:00:00.000Z | 1989-01-01T00:00:00.000Z +18 | 1989-01-01T00:00:00.000Z | 1988-01-01T00:00:00.000Z +30 | 1988-01-01T00:00:00.000Z | 1987-01-01T00:00:00.000Z +22 | 1987-01-01T00:00:00.000Z | 1986-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z +; + +MultiTypedBucketDateNanosInBothStatsAndByWithAlias +required_capability: implicit_casting_union_typed_numeric_and_date + +FROM employees, employees_incompatible +| STATS c = count(*), b = yr + 1 year BY yr = BUCKET(hire_date, 1 year) +| SORT yr DESC, c +; + +c:long | b:date_nanos | yr:date_nanos +2 | 2000-01-01T00:00:00.000Z | 1999-01-01T00:00:00.000Z +2 | 1998-01-01T00:00:00.000Z | 1997-01-01T00:00:00.000Z +2 | 1997-01-01T00:00:00.000Z | 1996-01-01T00:00:00.000Z +10 | 1996-01-01T00:00:00.000Z | 1995-01-01T00:00:00.000Z +8 | 1995-01-01T00:00:00.000Z | 1994-01-01T00:00:00.000Z +6 | 1994-01-01T00:00:00.000Z | 1993-01-01T00:00:00.000Z +16 | 1993-01-01T00:00:00.000Z | 1992-01-01T00:00:00.000Z +12 | 1992-01-01T00:00:00.000Z | 1991-01-01T00:00:00.000Z +24 | 1991-01-01T00:00:00.000Z | 1990-01-01T00:00:00.000Z +26 | 1990-01-01T00:00:00.000Z | 1989-01-01T00:00:00.000Z +18 | 1989-01-01T00:00:00.000Z | 1988-01-01T00:00:00.000Z +30 | 1988-01-01T00:00:00.000Z | 1987-01-01T00:00:00.000Z +22 | 1987-01-01T00:00:00.000Z | 1986-01-01T00:00:00.000Z +22 | 1986-01-01T00:00:00.000Z | 1985-01-01T00:00:00.000Z +; + +MultiTypedEnrichOnNumericField +required_capability: enrich_load +required_capability: implicit_casting_union_typed_numeric_and_date + +from employees, employees_incompatible +| enrich languages_policy on languages +| keep emp_no, language_name +| sort emp_no +| limit 6 +; + +emp_no:long |language_name:keyword +10001 | French +10001 | French +10002 | null +10002 | null +10003 | German +10003 | German +; + +MultiTypedEnrichOnNumericFieldWithEval +required_capability: enrich_load +required_capability: implicit_casting_union_typed_numeric_and_date + +from employees, employees_incompatible +| eval languages = languages + 1 +| enrich languages_policy on languages +| keep emp_no, language_name +| sort emp_no +| limit 6 +; + +emp_no:long |language_name:keyword +10001 | Spanish +10001 | Spanish +10002 | null +10002 | null +10003 | null +10003 | null +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 2ece0fdc92d11..edc1b3fbf15c3 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -308,6 +308,11 @@ public enum Cap { */ STRING_LITERAL_AUTO_CASTING_TO_DATETIME_ADD_SUB, + /** + * Support implicit casting for union typed numeric and date/date_nanos fields + */ + IMPLICIT_CASTING_UNION_TYPED_NUMERIC_AND_DATE, + /** * Support for named or positional parameters in EsqlQueryRequest. */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java index 980350ce43d4e..c3c2739d70cad 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java @@ -11,6 +11,7 @@ import org.elasticsearch.common.logging.LoggerMessageFormat; import org.elasticsearch.compute.data.Block; import org.elasticsearch.core.Strings; +import org.elasticsearch.core.Tuple; import org.elasticsearch.index.IndexMode; import org.elasticsearch.logging.Logger; import org.elasticsearch.xpack.core.enrich.EnrichPolicy; @@ -47,11 +48,13 @@ import org.elasticsearch.xpack.esql.core.util.Holder; import org.elasticsearch.xpack.esql.core.util.StringUtils; import org.elasticsearch.xpack.esql.expression.NamedExpressions; +import org.elasticsearch.xpack.esql.expression.Order; import org.elasticsearch.xpack.esql.expression.UnresolvedNamePattern; import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry; import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition; import org.elasticsearch.xpack.esql.expression.function.UnresolvedFunction; import org.elasticsearch.xpack.esql.expression.function.UnsupportedAttribute; +import org.elasticsearch.xpack.esql.expression.function.fulltext.FullTextFunction; import org.elasticsearch.xpack.esql.expression.function.grouping.GroupingFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.EsqlScalarFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Case; @@ -59,6 +62,8 @@ import org.elasticsearch.xpack.esql.expression.function.scalar.conditional.Least; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.AbstractConvertFunction; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.FoldablesConvertFunction; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDateNanos; +import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDatetime; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToDouble; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToInteger; import org.elasticsearch.xpack.esql.expression.function.scalar.convert.ToLong; @@ -85,6 +90,7 @@ import org.elasticsearch.xpack.esql.plan.logical.LogicalPlan; import org.elasticsearch.xpack.esql.plan.logical.Lookup; import org.elasticsearch.xpack.esql.plan.logical.MvExpand; +import org.elasticsearch.xpack.esql.plan.logical.OrderBy; import org.elasticsearch.xpack.esql.plan.logical.Project; import org.elasticsearch.xpack.esql.plan.logical.Rename; import org.elasticsearch.xpack.esql.plan.logical.RrfScoreEval; @@ -128,6 +134,7 @@ import static java.util.Collections.emptyList; import static java.util.Collections.singletonList; import static org.elasticsearch.xpack.core.enrich.EnrichPolicy.GEO_MATCH_TYPE; +import static org.elasticsearch.xpack.esql.analysis.Analyzer.ImplicitCasting.castInvalidMappedField; import static org.elasticsearch.xpack.esql.core.type.DataType.BOOLEAN; import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; @@ -140,9 +147,13 @@ import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.KEYWORD; import static org.elasticsearch.xpack.esql.core.type.DataType.LONG; +import static org.elasticsearch.xpack.esql.core.type.DataType.NULL; import static org.elasticsearch.xpack.esql.core.type.DataType.TEXT; import static org.elasticsearch.xpack.esql.core.type.DataType.TIME_DURATION; +import static org.elasticsearch.xpack.esql.core.type.DataType.UNSUPPORTED; import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION; +import static org.elasticsearch.xpack.esql.core.type.DataType.isMillisOrNanos; +import static org.elasticsearch.xpack.esql.core.type.DataType.isRepresentable; import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount; import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount; @@ -155,7 +166,7 @@ public class Analyzer extends ParameterizedRuleExecutor NO_FIELDS = List.of( - new ReferenceAttribute(Source.EMPTY, NO_FIELDS_NAME, DataType.NULL, Nullability.TRUE, null, true) + new ReferenceAttribute(Source.EMPTY, NO_FIELDS_NAME, NULL, Nullability.TRUE, null, true) ); private static final List> RULES = List.of( @@ -177,6 +188,7 @@ public class Analyzer extends ParameterizedRuleExecutor("Finish Analysis", Limiter.ONCE, new AddImplicitLimit(), new AddImplicitForkLimit(), new UnionTypesCleanup()) @@ -649,7 +661,7 @@ private LogicalPlan resolveLookup(Lookup l, List childrenOutput) { */ boolean dataTypesOk = joinedAttribute.dataType().equals(attr.dataType()); if (false == dataTypesOk) { - dataTypesOk = joinedAttribute.dataType() == DataType.NULL || attr.dataType() == DataType.NULL; + dataTypesOk = joinedAttribute.dataType() == NULL || attr.dataType() == NULL; } if (false == dataTypesOk) { dataTypesOk = joinedAttribute.dataType().equals(KEYWORD) && attr.dataType().equals(TEXT); @@ -1145,7 +1157,8 @@ private LogicalPlan resolveEnrich(Enrich enrich, List childrenOutput) final DataType dataType = resolved.dataType(); String matchType = enrich.policy().getType(); DataType[] allowed = allowedEnrichTypes(matchType); - if (Arrays.asList(allowed).contains(dataType) == false) { + // leave multi-typed fields to ImplicitCasting to cast to a common type + if (Arrays.asList(allowed).contains(dataType) == false && multiTypedField(resolved) == null) { String suffix = "only [" + Arrays.stream(allowed).map(DataType::typeName).collect(Collectors.joining(", ")) + "] allowed for type [" @@ -1302,6 +1315,63 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { return b; } + private static class ImplicitToExplicitCasting extends ParameterizedRule { + @Override + public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { + Map> invalidMappedFields = new HashMap<>(); + plan.forEachUp(LogicalPlan.class, p -> { + if (p instanceof EsRelation == false) { + p.forEachExpression(Attribute.class, a -> { + if (a instanceof FieldAttribute field) { + if (field.field() instanceof InvalidMappedField invalidMappedField) { + invalidMappedFields.computeIfAbsent(field, k -> new ArrayList<>()).add(p); + } + } + }); + } + }); + Map> planInvalidMapFields = new HashMap<>(); + invalidMappedFields.forEach((invalidMappedField, plans) -> { + for (LogicalPlan plan1 : plans) { + planInvalidMapFields.computeIfAbsent(plan1, k -> new HashSet<>()).add(invalidMappedField); + } + }); + Map> invalidMappedFieldCasted = new HashMap<>(); + LogicalPlan newPlan = plan.transformDown(LogicalPlan.class, p -> { + if (planInvalidMapFields.containsKey(p)) { + // If we are at a plan node that has invalid mapped fields, we need to either add an EVAL, or if that has been done + // we should instead replace with the already casted field + for (FieldAttribute invalidMappedField : planInvalidMapFields.get(p)) { + if (invalidMappedFieldCasted.containsKey(invalidMappedField) == false) { + // Add EVAL to cast the invalid mapped field + DataType targetType = targetType((InvalidMappedField) invalidMappedField.field()); + Expression conversionFunction = castInvalidMappedField(targetType, invalidMappedField); + p = new Eval(p.source(), p, List.of(new Alias(p.source(), invalidMappedField.name(), conversionFunction))); + // TODO: Also replace children + invalidMappedFieldCasted.put(invalidMappedField, new ArrayList<>()); + } else { + // Replace with already casted field + } + } + } + return p; + }); + return plan; + } + + DataType targetType(InvalidMappedField multiTypedField) { + DataType targetType = null; + for (DataType type : multiTypedField.types()) { + if (targetType == null) { // Initialize the target type to the first type. + targetType = type; + } else { + targetType = EsqlDataTypeConverter.commonType(targetType, type); + } + } + return targetType; + } + } + /** * Cast string literals in ScalarFunction, EsqlArithmeticOperation, BinaryComparison, In and GroupingFunction to desired data types. * For example, the string literals in the following expressions will be cast implicitly to the field data type on the left hand side. @@ -1315,7 +1385,7 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { *
  • date_trunc("1 minute", dateField)
    • * * If the inputs to Coalesce are mixed numeric types, cast the rest of the numeric field or value to the first numeric data type if - * applicable. For example, implicit casting converts: + * applicable, the same applies to Case, Greatest, Least. For example, implicit casting converts: *
      *
    • Coalesce(Long, Int) to Coalesce(Long, Long)
      • *
    • Coalesce(null, Long, Int) to Coalesce(null, Long, Long)
      • @@ -1324,9 +1394,30 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) { *
    * Coalesce(Int, Long) will NOT be converted to Coalesce(Long, Long) or Coalesce(Int, Int). */ - private static class ImplicitCasting extends ParameterizedRule { + static class ImplicitCasting extends ParameterizedRule { @Override public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { + // Do implicit casting for union typed fields in the following commands + /* + LogicalPlan newPlan = plan.transformUp(p -> { + if (p instanceof OrderBy + || p instanceof EsqlProject + || p instanceof Aggregate + || p instanceof MvExpand + || p instanceof Eval + || p instanceof LookupJoin + || p instanceof Enrich) { + return castInvalidMappedFieldInLogicalPlan(p, false); + } + return p; + }); + + // Add an implicit EsqlProject on top of the whole query if there isn't one yet, + // without explicit or implicit casting, a union typed field is returned as a null. + newPlan = addImplicitProjectForInvalidMappedFields(newPlan); + */ + + // do implicit casting for function arguments return plan.transformExpressionsUp( org.elasticsearch.xpack.esql.core.expression.function.Function.class, e -> ImplicitCasting.cast(e, context.functionRegistry().snapshotRegistry()) @@ -1334,6 +1425,9 @@ public LogicalPlan apply(LogicalPlan plan, AnalyzerContext context) { } private static Expression cast(org.elasticsearch.xpack.esql.core.expression.function.Function f, EsqlFunctionRegistry registry) { + // Add cast functions to InvalidMappedField if there isn't one yet + // f = castInvalidMappedFieldInFunction(f); + if (f instanceof In in) { return processIn(in); } @@ -1346,6 +1440,232 @@ private static Expression cast(org.elasticsearch.xpack.esql.core.expression.func return f; } + /** + * Both the logical plan's children and the logical plan itself can be changed, so we cannot just to a replaceChild here. + */ + private static LogicalPlan castInvalidMappedFieldInLogicalPlan(LogicalPlan plan, boolean addProject) { + List originalExpressions; + if (addProject) { + originalExpressions = plan.output(); + } else { + originalExpressions = switch (plan) { + case EsqlProject project -> project.projections(); + case OrderBy ob -> ob.order(); + case Aggregate agg -> agg.groupings(); + case MvExpand me -> List.of(me.target()); + case Eval eval -> eval.fields(); + case LookupJoin lj -> lj.config().leftFields(); + case Enrich enrich -> List.of(enrich.matchField()); + // The other types of plans are unexpected + default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan); + }; + } + Tuple, List> newChildren = castInvalidMappedFields(originalExpressions, true); + List aliases = newChildren.v1(); + List newProjections = newChildren.v2(); + if (aliases.isEmpty() == false) { + if (addProject) { + return esqlProjectForInvalidMappedField(plan.source(), plan, aliases, newProjections); + } + switch (plan) { + case EsqlProject p -> { + return esqlProjectForInvalidMappedField(p.source(), p.child(), aliases, newProjections); + } + case OrderBy o -> { + return new OrderBy( + o.source(), + o.child(), + newProjections.stream().filter(e -> e instanceof Order).map(e -> (Order) e).collect(Collectors.toList()) + ); + } + case Aggregate agg -> { + // Both groupings and aggregates need to be updated, create new aggregates according to new groupings + List origAggs = agg.aggregates(); + List newAggs = new ArrayList<>(origAggs.size()); + for (int i = 0; i < origAggs.size() - newProjections.size(); i++) { // Add aggregate functions + newAggs.add(origAggs.get(i)); + } + for (Expression e : newProjections) { // Add groupings + newAggs.add(Expressions.attribute(e)); + } + return agg.with(evalForInvalidMappedField(agg.source(), agg.child(), aliases), newProjections, newAggs); + } + case MvExpand mve -> { + NamedExpression newTarget = Expressions.attribute(newProjections.get(0)); + return new MvExpand( + mve.source(), + evalForInvalidMappedField(mve.source(), mve.child(), aliases), + newTarget, + newTarget.toAttribute() + ); + } + case Eval ev -> { + return evalForInvalidMappedField(ev.source(), ev.child(), aliases); + } + case LookupJoin lj -> { + JoinConfig oldJoinConfig = lj.config(); + List leftKeys = newProjections.stream() + .filter(e -> e instanceof Attribute) + .map(e -> (Attribute) e) + .collect(Collectors.toList()); + JoinConfig newJoinConfig = new JoinConfig(oldJoinConfig.type(), leftKeys, leftKeys, oldJoinConfig.rightFields()); + return new LookupJoin( + lj.source(), + evalForInvalidMappedField(lj.source(), lj.left(), aliases), + lj.right(), + newJoinConfig + ); + } + case Enrich enrich -> { + NamedExpression newMatchField = Expressions.attribute(newProjections.get(0)); + return new Enrich( + enrich.source(), + evalForInvalidMappedField(enrich.source(), enrich.child(), aliases), + enrich.mode(), + enrich.policyName(), + // Let resolveEnrich check whether the data type is supported, e.g. Enrich does not support data_nanos + new UnresolvedAttribute(newMatchField.source(), newMatchField.name()), + enrich.policy(), + enrich.concreteIndices(), + enrich.enrichFields() + ); + } + // The other types of plans are unexpected + default -> throw new EsqlIllegalArgumentException("unexpected logical plan: " + plan); + } + } else { + // Double check Aggs, when Bucket is used together with Aggs, the Aggs may need two rounds of resolutions + if (plan instanceof Aggregate agg) { + // If there is unresolved reference in the aggregates try to get it resolved again by removing the custom message + List origAggs = agg.aggregates(); + List groupings = agg.groupings(); + int aggsCount = origAggs.size() - groupings.size(); + List newAggs = new ArrayList<>(origAggs.size()); + for (int i = 0; i < origAggs.size(); i++) { + var e = origAggs.get(i); + if (i < aggsCount) { // Add aggregate functions + newAggs.add(e); + } else { // Add groupings + if (e instanceof UnresolvedAttribute ua && ua.customMessage()) { + // Try to make ResolveRefs resolve the aggregates again by removing the custom message + newAggs.add(Expressions.attribute(groupings.get(i - aggsCount))); + } else { + newAggs.add(e); + } + } + } + return agg.with(groupings, newAggs); + } + return plan; + } + } + + private static LogicalPlan addImplicitProjectForInvalidMappedFields(LogicalPlan logicalPlan) { + if (logicalPlan.resolved() == false) { + return logicalPlan; + } + List projections = logicalPlan.collectFirstChildren(EsqlProject.class::isInstance); + return projections.isEmpty() ? castInvalidMappedFieldInLogicalPlan(logicalPlan, true) : logicalPlan; + } + + private static org.elasticsearch.xpack.esql.core.expression.function.Function castInvalidMappedFieldInFunction( + org.elasticsearch.xpack.esql.core.expression.function.Function f + ) { + // No need to add implicit casting for union typed fields that already have explicit casting on them. + // Full text functions are pushdown only functions, implicit or explicit casting may fail the query. + if (f instanceof AbstractConvertFunction || f instanceof FullTextFunction) { + return f; + } + Tuple, List> newChildren = castInvalidMappedFields(f.arguments(), false); + return newChildren.v1().isEmpty() + ? f + : (org.elasticsearch.xpack.esql.core.expression.function.Function) f.replaceChildren(newChildren.v2()); + } + + private static Tuple, List> castInvalidMappedFields( + List originalExpressions, + boolean createNewChildPlan + ) { + List newAliases = new ArrayList<>(originalExpressions.size()); + List newExpressions = new ArrayList<>(originalExpressions.size()); + expressionLoop: for (Expression exp : originalExpressions) { + Expression e = Alias.unwrap(exp); + e = e instanceof Order o ? o.child() : e; + String alias = exp instanceof Alias a ? a.name() : null; + InvalidMappedField multiTypedField = multiTypedField(e); + if (multiTypedField != null) { + // This is an invalid mapped field, find a common data type and cast to it. + DataType targetType = null; + for (DataType type : multiTypedField.types()) { + if (targetType == null) { // Initialize the target type to the first type. + targetType = type; + } else { + targetType = EsqlDataTypeConverter.commonType(targetType, type); + if (targetType == null) { // If there is no common type, continue to the next expression. + newExpressions.add(exp); + continue expressionLoop; + } + } + } + if (targetType != null && isRepresentable(targetType) && (isMillisOrNanos(targetType) || targetType.isNumeric())) { + alias = alias != null ? alias : multiTypedField.getName(); + Source source = e.source(); + Expression newChild = castInvalidMappedField(targetType, e); + Alias newAlias = new Alias(source, alias, newChild); + newAliases.add(newAlias); + if (createNewChildPlan) { + // Cast union typed fields in a logical plan, a new child plan(Eval) is needed for the implicit casting and new + // aliases. The newExpressions with all the fields and references are need to create a new plan. + switch (exp) { + case Alias a -> newExpressions.add(a.replaceChild(newAlias.toAttribute())); + case Order o -> newExpressions.add(new Order(o.source(), newChild, o.direction(), o.nullsPosition())); + default -> newExpressions.add(newAlias.toAttribute()); + } + } else { // Cast union typed fields in a function, there is no need to create a new child plan + newExpressions.add(newChild); + } + } + } else { + newExpressions.add(exp); + } + } + return Tuple.tuple(newAliases, newExpressions); + } + + private static EsqlProject esqlProjectForInvalidMappedField( + Source source, + LogicalPlan childPlan, + List aliases, + List newProjections + ) { + Eval eval = evalForInvalidMappedField(source, childPlan, aliases); + return new EsqlProject( + source, + eval, + newProjections.stream().filter(e -> e instanceof NamedExpression).map(e -> (NamedExpression) e).collect(Collectors.toList()) + ); + } + + private static Eval evalForInvalidMappedField(Source source, LogicalPlan childPlan, List aliases) { + return new Eval(source, childPlan, aliases); + } + + /** + * Do implicit casting for data, date_nanos and numeric types only + */ + static Expression castInvalidMappedField(DataType targetType, Expression fa) { + Source source = fa.source(); + return switch (targetType) { + case INTEGER -> new ToInteger(source, fa); + case LONG -> new ToLong(source, fa); + case DOUBLE -> new ToDouble(source, fa); + case UNSIGNED_LONG -> new ToUnsignedLong(source, fa); + case DATETIME -> new ToDatetime(source, fa); + case DATE_NANOS -> new ToDateNanos(source, fa); + default -> throw new EsqlIllegalArgumentException("unexpected data type: " + targetType); + }; + } + private static Expression processScalarOrGroupingFunction( org.elasticsearch.xpack.esql.core.expression.function.Function f, EsqlFunctionRegistry registry @@ -1357,7 +1677,7 @@ private static Expression processScalarOrGroupingFunction( } List newChildren = new ArrayList<>(args.size()); boolean childrenChanged = false; - DataType targetDataType = DataType.NULL; + DataType targetDataType = NULL; Expression arg; DataType targetNumericType = null; boolean castNumericArgs = true; @@ -1370,7 +1690,7 @@ private static Expression processScalarOrGroupingFunction( if (i < targetDataTypes.size()) { targetDataType = targetDataTypes.get(i); } - if (targetDataType != DataType.NULL && targetDataType != DataType.UNSUPPORTED) { + if (targetDataType != NULL && targetDataType != UNSUPPORTED) { Expression e = castStringLiteral(arg, targetDataType); if (e != arg) { childrenChanged = true; @@ -1403,7 +1723,7 @@ private static Expression processBinaryOperator(BinaryOperator o) { } List newChildren = new ArrayList<>(2); boolean childrenChanged = false; - DataType targetDataType = DataType.NULL; + DataType targetDataType = NULL; Expression from = Literal.NULL; if (left.dataType() == KEYWORD && left.foldable() && (left instanceof EsqlScalarFunction == false)) { @@ -1747,4 +2067,15 @@ private static LogicalPlan planWithoutSyntheticAttributes(LogicalPlan plan) { return newOutput.size() == output.size() ? plan : new Project(Source.EMPTY, plan, newOutput); } } + + private static InvalidMappedField multiTypedField(Expression e) { + Expression exp = Alias.unwrap(e); + if (exp.resolved() + && exp.dataType() == UNSUPPORTED + && exp instanceof FieldAttribute fa + && fa.field() instanceof InvalidMappedField imf) { + return imf; + } + return null; + } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java index 11e9a57064e5b..e9defff9e6525 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Enrich.java @@ -32,6 +32,7 @@ import org.elasticsearch.xpack.esql.core.expression.ReferenceAttribute; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.index.EsIndex; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import org.elasticsearch.xpack.esql.plan.GeneratingPlan; @@ -208,6 +209,7 @@ public boolean expressionsResolved() { return policyName.resolved() && matchField instanceof EmptyAttribute == false // matchField not defined in the query, needs to be resolved from the policy && matchField.resolved() + && matchField.dataType() != DataType.UNSUPPORTED && Resolvables.resolved(enrichFields()); } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java index 5c0aa35f13880..90e35dfc73af9 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/Eval.java @@ -13,7 +13,6 @@ import org.elasticsearch.xpack.esql.capabilities.PostAnalysisVerificationAware; import org.elasticsearch.xpack.esql.capabilities.TelemetryAware; import org.elasticsearch.xpack.esql.common.Failures; -import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; import org.elasticsearch.xpack.esql.core.expression.Alias; import org.elasticsearch.xpack.esql.core.expression.Attribute; import org.elasticsearch.xpack.esql.core.expression.AttributeMap; @@ -132,7 +131,12 @@ private List renameAliases(List originalAttributes, List n @Override public boolean expressionsResolved() { - return Resolvables.resolved(fields); + for (Alias a : fields) { + if (a.resolved() == false || a.dataType() == DataType.UNSUPPORTED) { + return false; + } + } + return true; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java index f65811fc26526..52941018ffae8 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/MvExpand.java @@ -16,6 +16,7 @@ import org.elasticsearch.xpack.esql.core.expression.NamedExpression; import org.elasticsearch.xpack.esql.core.tree.NodeInfo; import org.elasticsearch.xpack.esql.core.tree.Source; +import org.elasticsearch.xpack.esql.core.type.DataType; import org.elasticsearch.xpack.esql.io.stream.PlanStreamInput; import java.io.IOException; @@ -90,7 +91,7 @@ public String telemetryLabel() { @Override public boolean expressionsResolved() { - return target.resolved(); + return target.resolved() && target.dataType() != DataType.UNSUPPORTED; } @Override diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java index 383606d6ccbed..d9c5a9fc2d063 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/plan/logical/join/JoinConfig.java @@ -12,6 +12,7 @@ import org.elasticsearch.common.io.stream.Writeable; import org.elasticsearch.xpack.esql.core.capabilities.Resolvables; import org.elasticsearch.xpack.esql.core.expression.Attribute; +import org.elasticsearch.xpack.esql.core.type.DataType; import java.io.IOException; import java.util.List; @@ -50,6 +51,7 @@ public boolean expressionsResolved() { return type.resolved() && Resolvables.resolved(matchFields) && Resolvables.resolved(leftFields) - && Resolvables.resolved(rightFields); + && Resolvables.resolved(rightFields) + && leftFields.stream().noneMatch(e -> e.dataType() == DataType.UNSUPPORTED); } } diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java index 11a8c5c5f5b98..21d4800165095 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverter.java @@ -404,6 +404,9 @@ public static DataType commonType(DataType left, DataType right) { if (isNullOrDatePeriod(left) && isNullOrDatePeriod(right)) { return DATE_PERIOD; } + if ((isDateTime(left) && right == DATE_NANOS) || (left == DATE_NANOS && isDateTime(right))) { + return DATE_NANOS; + } } if (isString(left) && isString(right)) { // Both TEXT and SEMANTIC_TEXT are processed as KEYWORD diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java index c1416d9f83b55..72f617a450845 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/analysis/VerifierTests.java @@ -133,8 +133,8 @@ public void testUnsupportedAndMultiTypedFields() { error("from test* | enrich client_cidr on unsupported", analyzer) ); assertEquals( - "1:36: Unsupported type [unsupported] for enrich matching field [multi_typed];" - + " only [keyword, text, ip, long, integer, float, double, datetime] allowed for type [range]", + "1:36: Cannot use field [multi_typed] due to ambiguities being mapped as [2] incompatible types: " + + "[ip] in [test1, test2, test3] and [2] other indices, [keyword] in [test6]", error("from test* | enrich client_cidr on multi_typed", analyzer) ); diff --git a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java index 9a30c2281d742..487a2ee111c03 100644 --- a/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java +++ b/x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/type/EsqlDataTypeConverterTests.java @@ -97,7 +97,7 @@ public void testCommonTypeDateTimeIntervals() { assertEqualsCommonType(dataType1, NULL, dataType1); } else if (isDateTimeOrNanosOrTemporal(dataType2)) { if ((dataType1 == DATE_NANOS && dataType2 == DATETIME) || (dataType1 == DATETIME && dataType2 == DATE_NANOS)) { - assertNullCommonType(dataType1, dataType2); + assertEqualsCommonType(dataType1, dataType2, DATE_NANOS); } else if (isDateTime(dataType1) || isDateTime(dataType2)) { assertEqualsCommonType(dataType1, dataType2, DATETIME); } else if (dataType1 == DATE_NANOS || dataType2 == DATE_NANOS) {