Skip to content

Commit c5116cc

Browse files
not-napoleonelasticsearchmachine
authored and
omricohenn
committed
Esql - Fix lucene push down behavior when a range contains nanos and millis (elastic#125595)
Follow up to elastic#125345. If the query contained both a nanos and a millis comparison, we were formatting the dates incorrectly for the lucene push down. This PR adds a test and a fix for that case. --------- Co-authored-by: elasticsearchmachine <[email protected]>
1 parent 71237d9 commit c5116cc

File tree

4 files changed

+133
-10
lines changed

4 files changed

+133
-10
lines changed

‎docs/changelog/125595.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 125595
2+
summary: Esql - Fix lucene push down behavior when a range contains nanos and millis
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1762,3 +1762,110 @@ nanos:date_nanos
17621762
2023-10-23T12:15:03.360103847Z
17631763
2023-10-23T12:15:03.360103847Z
17641764
;
1765+
1766+
Range Without Included Endpoints
1767+
required_capability: to_date_nanos
1768+
required_capability: fix_date_nanos_lucene_pushdown_bug
1769+
1770+
FROM date_nanos
1771+
| WHERE millis > "2020-01-01"
1772+
| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datenanos("2023-10-23T13:53:55.832987654Z")
1773+
| KEEP nanos;
1774+
ignoreOrder:true
1775+
warningRegex:Line 3:67: evaluation of \[nanos < to_datenanos\(\\\"2023-10-23T13:53:55\.832987654Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1776+
warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value
1777+
warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1778+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1779+
1780+
nanos:date_nanos
1781+
2023-10-23T13:52:55.015787878Z
1782+
2023-10-23T13:51:54.732102837Z
1783+
2023-10-23T13:33:34.937193000Z
1784+
2023-10-23T12:27:28.948000000Z
1785+
;
1786+
1787+
Range Without Included Endpoints with implicit casting
1788+
required_capability: to_date_nanos
1789+
required_capability: fix_date_nanos_lucene_pushdown_bug
1790+
1791+
FROM date_nanos
1792+
| WHERE millis > "2020-01-01"
1793+
| WHERE nanos > "2023-10-23T12:15:03.360103847" AND nanos < "2023-10-23T13:53:55.832987654Z"
1794+
| KEEP nanos;
1795+
ignoreOrder:true
1796+
warningRegex:Line 3:53: evaluation of \[nanos < \\\"2023-10-23T13:53:55\.832987654Z\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
1797+
warningRegex:Line 3:53: java.lang.IllegalArgumentException: single-value function encountered multi-value
1798+
warningRegex:Line 3:9: evaluation of \[nanos > \\\"2023-10-23T12:15:03\.360103847\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
1799+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1800+
1801+
nanos:date_nanos
1802+
2023-10-23T13:52:55.015787878Z
1803+
2023-10-23T13:51:54.732102837Z
1804+
2023-10-23T13:33:34.937193000Z
1805+
2023-10-23T12:27:28.948000000Z
1806+
;
1807+
1808+
Range With Now date math
1809+
required_capability: to_date_nanos
1810+
required_capability: fix_date_nanos_lucene_pushdown_bug
1811+
1812+
FROM date_nanos
1813+
| WHERE millis > "2020-01-01"
1814+
| WHERE nanos > TO_DATETIME("2023-10-23T12:27:28.948") AND nanos < now() - 1d
1815+
| KEEP nanos;
1816+
ignoreOrder:true
1817+
warningRegex:Line 3:60: evaluation of \[nanos < now\(\) - 1d\] failed, treating result as null\. Only first 20 failures recorded\.
1818+
warningRegex:Line 3:60: java.lang.IllegalArgumentException: single-value function encountered multi-value
1819+
warningRegex:Line 3:9: evaluation of \[nanos > TO_DATETIME\(\\\"2023-10-23T12:27:28\.948\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1820+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1821+
1822+
nanos:date_nanos
1823+
2023-10-23T13:55:01.543123456Z
1824+
2023-10-23T13:53:55.832987654Z
1825+
2023-10-23T13:52:55.015787878Z
1826+
2023-10-23T13:51:54.732102837Z
1827+
2023-10-23T13:33:34.937193000Z
1828+
;
1829+
1830+
Mixed Nanos Millis Range Compare
1831+
required_capability: to_date_nanos
1832+
required_capability: fix_date_nanos_lucene_pushdown_bug
1833+
required_capability: fix_date_nanos_mixed_range_pushdown_bug
1834+
1835+
FROM date_nanos
1836+
| WHERE millis > "2020-01-01"
1837+
| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datetime("2023-10-23T13:53:55.832")
1838+
| KEEP nanos;
1839+
ignoreOrder:true
1840+
warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1841+
warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value
1842+
warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1843+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1844+
1845+
nanos:date_nanos
1846+
2023-10-23T13:52:55.015787878Z
1847+
2023-10-23T13:51:54.732102837Z
1848+
2023-10-23T13:33:34.937193000Z
1849+
2023-10-23T12:27:28.948000000Z
1850+
;
1851+
1852+
Mixed Nanos Millis Range Compare, millis field
1853+
required_capability: to_date_nanos
1854+
required_capability: fix_date_nanos_lucene_pushdown_bug
1855+
required_capability: fix_date_nanos_mixed_range_pushdown_bug
1856+
1857+
FROM date_nanos
1858+
| WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")
1859+
| KEEP nanos;
1860+
ignoreOrder:true
1861+
warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1862+
warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value
1863+
warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1864+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1865+
1866+
nanos:date_nanos
1867+
2023-10-23T13:52:55.015787878Z
1868+
2023-10-23T13:51:54.732102837Z
1869+
2023-10-23T13:33:34.937193000Z
1870+
2023-10-23T12:27:28.948000000Z
1871+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -554,6 +554,11 @@ public enum Cap {
554554
* Indicates that https://github.com/elastic/elasticsearch/issues/125439 (incorrect lucene push down for date nanos) is fixed
555555
*/
556556
FIX_DATE_NANOS_LUCENE_PUSHDOWN_BUG(),
557+
/**
558+
* Fixes a bug where dates are incorrectly formatted if a where clause compares nanoseconds to both milliseconds and nanoseconds,
559+
* e.g. {@code WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")}
560+
*/
561+
FIX_DATE_NANOS_MIXED_RANGE_PUSHDOWN_BUG(),
557562
/**
558563
* DATE_PARSE supports reading timezones
559564
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333

3434
import static java.util.Arrays.asList;
3535
import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf;
36-
import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
3736
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
3837
import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
3938
import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
@@ -42,9 +41,8 @@
4241
import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
4342
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER;
4443
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
45-
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
44+
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateWithTypeToString;
4645
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
47-
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
4846
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
4947

5048
// BETWEEN or range - is a mix of gt(e) AND lt(e)
@@ -217,16 +215,23 @@ private RangeQuery translate(TranslatorHandler handler) {
217215
String format = null;
218216

219217
DataType dataType = value.dataType();
220-
logger.trace("Translating Range into lucene query. dataType is [{}] upper is [{}] lower is [{}]", dataType, lower, upper);
221-
if (dataType == DataType.DATETIME && lower.dataType() == DATETIME && upper.dataType() == DATETIME) {
222-
l = dateTimeToString((Long) l);
223-
u = dateTimeToString((Long) u);
218+
logger.warn(
219+
"Translating Range into lucene query. dataType is [{}] upper is [{}<{}>] lower is [{}<{}>]",
220+
dataType,
221+
lower,
222+
lower.dataType(),
223+
upper,
224+
upper.dataType()
225+
);
226+
if (dataType == DataType.DATETIME) {
227+
l = dateWithTypeToString((Long) l, lower.dataType());
228+
u = dateWithTypeToString((Long) u, upper.dataType());
224229
format = DEFAULT_DATE_TIME_FORMATTER.pattern();
225230
}
226231

227-
if (dataType == DATE_NANOS && lower.dataType() == DATE_NANOS && upper.dataType() == DATE_NANOS) {
228-
l = nanoTimeToString((Long) l);
229-
u = nanoTimeToString((Long) u);
232+
if (dataType == DATE_NANOS) {
233+
l = dateWithTypeToString((Long) l, lower.dataType());
234+
u = dateWithTypeToString((Long) u, upper.dataType());
230235
format = DEFAULT_DATE_NANOS_FORMATTER.pattern();
231236
}
232237

@@ -258,6 +263,7 @@ private RangeQuery translate(TranslatorHandler handler) {
258263
u = unsignedLongAsNumber(ul);
259264
}
260265
}
266+
logger.warn("Building range query with format string [{}]", format);
261267
return new RangeQuery(source(), handler.nameOf(value), l, includeLower(), u, includeUpper(), format, zoneId);
262268
}
263269

0 commit comments

Comments
 (0)