Skip to content

Commit f728b9c

Browse files
not-napoleonelasticsearchmachine
not-napoleon
and
elasticsearchmachine
committed
Esql - Fix lucene push down behavior when a range contains nanos and millis (elastic#125595)
Follow up to elastic#125345. If the query contained both a nanos and a millis comparison, we were formatting the dates incorrectly for the lucene push down. This PR adds a test and a fix for that case. --------- Co-authored-by: elasticsearchmachine <[email protected]>
1 parent abcec9e commit f728b9c

File tree

4 files changed

+133
-10
lines changed

4 files changed

+133
-10
lines changed

‎docs/changelog/125595.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 125595
2+
summary: Esql - Fix lucene push down behavior when a range contains nanos and millis
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec‎

Lines changed: 107 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1754,3 +1754,110 @@ nanos:date_nanos
17541754
2023-10-23T12:15:03.360103847Z
17551755
2023-10-23T12:15:03.360103847Z
17561756
;
1757+
1758+
Range Without Included Endpoints
1759+
required_capability: to_date_nanos
1760+
required_capability: fix_date_nanos_lucene_pushdown_bug
1761+
1762+
FROM date_nanos
1763+
| WHERE millis > "2020-01-01"
1764+
| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datenanos("2023-10-23T13:53:55.832987654Z")
1765+
| KEEP nanos;
1766+
ignoreOrder:true
1767+
warningRegex:Line 3:67: evaluation of \[nanos < to_datenanos\(\\\"2023-10-23T13:53:55\.832987654Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1768+
warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value
1769+
warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1770+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1771+
1772+
nanos:date_nanos
1773+
2023-10-23T13:52:55.015787878Z
1774+
2023-10-23T13:51:54.732102837Z
1775+
2023-10-23T13:33:34.937193000Z
1776+
2023-10-23T12:27:28.948000000Z
1777+
;
1778+
1779+
Range Without Included Endpoints with implicit casting
1780+
required_capability: to_date_nanos
1781+
required_capability: fix_date_nanos_lucene_pushdown_bug
1782+
1783+
FROM date_nanos
1784+
| WHERE millis > "2020-01-01"
1785+
| WHERE nanos > "2023-10-23T12:15:03.360103847" AND nanos < "2023-10-23T13:53:55.832987654Z"
1786+
| KEEP nanos;
1787+
ignoreOrder:true
1788+
warningRegex:Line 3:53: evaluation of \[nanos < \\\"2023-10-23T13:53:55\.832987654Z\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
1789+
warningRegex:Line 3:53: java.lang.IllegalArgumentException: single-value function encountered multi-value
1790+
warningRegex:Line 3:9: evaluation of \[nanos > \\\"2023-10-23T12:15:03\.360103847\\\"\] failed, treating result as null\. Only first 20 failures recorded\.
1791+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1792+
1793+
nanos:date_nanos
1794+
2023-10-23T13:52:55.015787878Z
1795+
2023-10-23T13:51:54.732102837Z
1796+
2023-10-23T13:33:34.937193000Z
1797+
2023-10-23T12:27:28.948000000Z
1798+
;
1799+
1800+
Range With Now date math
1801+
required_capability: to_date_nanos
1802+
required_capability: fix_date_nanos_lucene_pushdown_bug
1803+
1804+
FROM date_nanos
1805+
| WHERE millis > "2020-01-01"
1806+
| WHERE nanos > TO_DATETIME("2023-10-23T12:27:28.948") AND nanos < now() - 1d
1807+
| KEEP nanos;
1808+
ignoreOrder:true
1809+
warningRegex:Line 3:60: evaluation of \[nanos < now\(\) - 1d\] failed, treating result as null\. Only first 20 failures recorded\.
1810+
warningRegex:Line 3:60: java.lang.IllegalArgumentException: single-value function encountered multi-value
1811+
warningRegex:Line 3:9: evaluation of \[nanos > TO_DATETIME\(\\\"2023-10-23T12:27:28\.948\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1812+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1813+
1814+
nanos:date_nanos
1815+
2023-10-23T13:55:01.543123456Z
1816+
2023-10-23T13:53:55.832987654Z
1817+
2023-10-23T13:52:55.015787878Z
1818+
2023-10-23T13:51:54.732102837Z
1819+
2023-10-23T13:33:34.937193000Z
1820+
;
1821+
1822+
Mixed Nanos Millis Range Compare
1823+
required_capability: to_date_nanos
1824+
required_capability: fix_date_nanos_lucene_pushdown_bug
1825+
required_capability: fix_date_nanos_mixed_range_pushdown_bug
1826+
1827+
FROM date_nanos
1828+
| WHERE millis > "2020-01-01"
1829+
| WHERE nanos > to_datenanos("2023-10-23T12:15:03.360103847") AND nanos < to_datetime("2023-10-23T13:53:55.832")
1830+
| KEEP nanos;
1831+
ignoreOrder:true
1832+
warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1833+
warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value
1834+
warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1835+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1836+
1837+
nanos:date_nanos
1838+
2023-10-23T13:52:55.015787878Z
1839+
2023-10-23T13:51:54.732102837Z
1840+
2023-10-23T13:33:34.937193000Z
1841+
2023-10-23T12:27:28.948000000Z
1842+
;
1843+
1844+
Mixed Nanos Millis Range Compare, millis field
1845+
required_capability: to_date_nanos
1846+
required_capability: fix_date_nanos_lucene_pushdown_bug
1847+
required_capability: fix_date_nanos_mixed_range_pushdown_bug
1848+
1849+
FROM date_nanos
1850+
| WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")
1851+
| KEEP nanos;
1852+
ignoreOrder:true
1853+
warningRegex:Line 3:67: evaluation of \[nanos < to_datetime\(\\\"2023-10-23T13:53:55\.832Z\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1854+
warningRegex:Line 3:67: java.lang.IllegalArgumentException: single-value function encountered multi-value
1855+
warningRegex:Line 3:9: evaluation of \[nanos > to_datenanos\(\\\"2023-10-23T12:15:03\.360103847\\\"\)\] failed, treating result as null\. Only first 20 failures recorded\.
1856+
warningRegex:Line 3:9: java.lang.IllegalArgumentException: single-value function encountered multi-value
1857+
1858+
nanos:date_nanos
1859+
2023-10-23T13:52:55.015787878Z
1860+
2023-10-23T13:51:54.732102837Z
1861+
2023-10-23T13:33:34.937193000Z
1862+
2023-10-23T12:27:28.948000000Z
1863+
;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -537,6 +537,11 @@ public enum Cap {
537537
* Indicates that https://github.com/elastic/elasticsearch/issues/125439 (incorrect lucene push down for date nanos) is fixed
538538
*/
539539
FIX_DATE_NANOS_LUCENE_PUSHDOWN_BUG(),
540+
/**
541+
* Fixes a bug where dates are incorrectly formatted if a where clause compares nanoseconds to both milliseconds and nanoseconds,
542+
* e.g. {@code WHERE millis > to_datenanos("2023-10-23T12:15:03.360103847") AND millis < to_datetime("2023-10-23T13:53:55.832")}
543+
*/
544+
FIX_DATE_NANOS_MIXED_RANGE_PUSHDOWN_BUG(),
540545
/**
541546
* DATE_PARSE supports reading timezones
542547
*/

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/expression/predicate/Range.java‎

Lines changed: 16 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,6 @@
3333

3434
import static java.util.Arrays.asList;
3535
import static org.elasticsearch.xpack.esql.core.expression.Foldables.valueOf;
36-
import static org.elasticsearch.xpack.esql.core.type.DataType.DATETIME;
3736
import static org.elasticsearch.xpack.esql.core.type.DataType.DATE_NANOS;
3837
import static org.elasticsearch.xpack.esql.core.type.DataType.IP;
3938
import static org.elasticsearch.xpack.esql.core.type.DataType.UNSIGNED_LONG;
@@ -42,9 +41,8 @@
4241
import static org.elasticsearch.xpack.esql.core.util.NumericUtils.unsignedLongAsNumber;
4342
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_NANOS_FORMATTER;
4443
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.DEFAULT_DATE_TIME_FORMATTER;
45-
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateTimeToString;
44+
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.dateWithTypeToString;
4645
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.ipToString;
47-
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.nanoTimeToString;
4846
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.versionToString;
4947

5048
// BETWEEN or range - is a mix of gt(e) AND lt(e)
@@ -217,16 +215,23 @@ private RangeQuery translate(TranslatorHandler handler) {
217215
String format = null;
218216

219217
DataType dataType = value.dataType();
220-
logger.trace("Translating Range into lucene query. dataType is [{}] upper is [{}] lower is [{}]", dataType, lower, upper);
221-
if (dataType == DataType.DATETIME && lower.dataType() == DATETIME && upper.dataType() == DATETIME) {
222-
l = dateTimeToString((Long) l);
223-
u = dateTimeToString((Long) u);
218+
logger.warn(
219+
"Translating Range into lucene query. dataType is [{}] upper is [{}<{}>] lower is [{}<{}>]",
220+
dataType,
221+
lower,
222+
lower.dataType(),
223+
upper,
224+
upper.dataType()
225+
);
226+
if (dataType == DataType.DATETIME) {
227+
l = dateWithTypeToString((Long) l, lower.dataType());
228+
u = dateWithTypeToString((Long) u, upper.dataType());
224229
format = DEFAULT_DATE_TIME_FORMATTER.pattern();
225230
}
226231

227-
if (dataType == DATE_NANOS && lower.dataType() == DATE_NANOS && upper.dataType() == DATE_NANOS) {
228-
l = nanoTimeToString((Long) l);
229-
u = nanoTimeToString((Long) u);
232+
if (dataType == DATE_NANOS) {
233+
l = dateWithTypeToString((Long) l, lower.dataType());
234+
u = dateWithTypeToString((Long) u, upper.dataType());
230235
format = DEFAULT_DATE_NANOS_FORMATTER.pattern();
231236
}
232237

@@ -258,6 +263,7 @@ private RangeQuery translate(TranslatorHandler handler) {
258263
u = unsignedLongAsNumber(ul);
259264
}
260265
}
266+
logger.warn("Building range query with format string [{}]", format);
261267
return new RangeQuery(source(), handler.nameOf(value), l, includeLower(), u, includeUpper(), format, zoneId);
262268
}
263269

0 commit comments

Comments
 (0)