diff --git a/docs/changelog/125595.yaml b/docs/changelog/125595.yaml new file mode 100644 index 0000000000000..2a3f4c61c07b0 --- /dev/null +++ b/docs/changelog/125595.yaml @@ -0,0 +1,5 @@ +pr: 125595 +summary: Esql - Fix lucene push down behavior when a range contains nanos and millis +area: ES|QL +type: bug +issues: [] diff --git a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec index b9d21d3b2856d..6ca3fb75f22b5 100644 --- a/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec +++ b/x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec @@ -1753,3 +1753,110 @@ nanos:date_nanos 2023-10-23T12:15:03.360103847Z 2023-10-23T12:15:03.360103847Z ; + +Range Without Included Endpoints +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE millis > "2020-01-01" +| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datenanos("2023-10-23T13:53:55.832987654Z") +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:67: evaluation of \[nanos < to_datenanos\(\\\"2023-10-23T13:53:55\.832987654Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +; + +Range Without Included Endpoints with implicit casting +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE millis > "2020-01-01" +| WHERE nanos > "2023-10-23T12:15:03.360103847" AND nanos < "2023-10-23T13:53:55.832987654Z" +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:53: evaluation of \[nanos < \\\"2023-10-23T13:53:55\.832987654Z\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:53: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > \\\"2023-10-23T12:15:03\.360103847\\\"\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +; + +Range With Now date math +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug + +FROM date_nanos +| WHERE millis > "2020-01-01" +| WHERE nanos > TO_DATETIME("2023-10-23T12:27:28.948") AND nanos < now() - 1d +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:60: evaluation of \[nanos < now\(\) - 1d\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:60: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > TO_DATETIME\(\\\"2023-10-23T12:27:28\.948\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:55:01.543123456Z +2023-10-23T13:53:55.832987654Z +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +; + +Mixed Nanos Millis Range Compare +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug +required_capability: fix_date_nanos_mixed_range_pushdown_bug + +FROM date_nanos +| WHERE millis > "2020-01-01" +| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datetime("2023-10-23T13:53:55.832") +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +; + +Mixed Nanos Millis Range Compare, millis field +required_capability: to_date_nanos +required_capability: fix_date_nanos_lucene_pushdown_bug +required_capability: fix_date_nanos_mixed_range_pushdown_bug + +FROM date_nanos +| WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832") +| KEEP nanos; +ignoreOrder:true +warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value +warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\. +warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value + +nanos:date_nanos +2023-10-23T13:52:55.015787878Z +2023-10-23T13:51:54.732102837Z +2023-10-23T13:33:34.937193000Z +2023-10-23T12:27:28.948000000Z +; diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java index 28df0a65d58b8..a654961879a61 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java @@ -402,6 +402,11 @@ public enum Cap { * Indicates that https://github.com/elastic/elasticsearch/issues/125439 (incorrect lucene push down for date nanos) is fixed */ FIX_DATE_NANOS_LUCENE_PUSHDOWN_BUG(), + /** + * Fixes a bug where dates are incorrectly formatted if a where clause compares nanoseconds to both milliseconds and nanoseconds, + * e.g. {@code WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")} + */ + FIX_DATE_NANOS_MIXED_RANGE_PUSHDOWN_BUG(), /** * DATE_PARSE supports reading timezones */ diff --git a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java index 4f9776f40d940..fd03165617ab6 100644 --- a/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java +++ b/x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java @@ -33,7 +33,6 @@ import static java.util.Arrays.asList; import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf; -import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME; import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS; import static org.elasticsearch.xpack.esql.core.type.DataType.IP; import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG; @@ -42,9 +41,8 @@ import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString; +import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateWithTypeToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString; -import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString; import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString; // BETWEEN or range - is a mix of gt(e) AND lt(e) @@ -217,16 +215,23 @@ private RangeQuery translate(TranslatorHandler handler) { String format = null; DataType dataType = value.dataType(); - logger.trace("Translating Range into lucene query. dataType is [{}] upper is [{}] lower is [{}]", dataType, lower, upper); - if (dataType == DataType.DATETIME && lower.dataType() == DATETIME && upper.dataType() == DATETIME) { - l = dateTimeToString((Long) l); - u = dateTimeToString((Long) u); + logger.warn( + "Translating Range into lucene query. dataType is [{}] upper is [{}<{}>] lower is [{}<{}>]", + dataType, + lower, + lower.dataType(), + upper, + upper.dataType() + ); + if (dataType == DataType.DATETIME) { + l = dateWithTypeToString((Long) l, lower.dataType()); + u = dateWithTypeToString((Long) u, upper.dataType()); format = DEFAULT_DATE_TIME_FORMATTER.pattern(); } - if (dataType == DATE_NANOS && lower.dataType() == DATE_NANOS && upper.dataType() == DATE_NANOS) { - l = nanoTimeToString((Long) l); - u = nanoTimeToString((Long) u); + if (dataType == DATE_NANOS) { + l = dateWithTypeToString((Long) l, lower.dataType()); + u = dateWithTypeToString((Long) u, upper.dataType()); format = DEFAULT_DATE_NANOS_FORMATTER.pattern(); } @@ -258,6 +263,7 @@ private RangeQuery translate(TranslatorHandler handler) { u = unsignedLongAsNumber(ul); } } + logger.warn("Building range query with format string [{}]", format); return new RangeQuery(source(), handler.nameOf(value), l, includeLower(), u, includeUpper(), format, zoneId); }