Skip to content

Commit 187a6a7

Browse files
not-napoleonelasticsearchmachine
andauthored
ESQL Support IN operator for Date nanos (elastic#119772)
Add support for using nanosecond dates with the IN operator. This behavior should be consistent with equals, and support comparisons between milliseconds and nanoseconds the same as the binary comparison operators support it. Resolves elastic#118578 --------- Co-authored-by: elasticsearchmachine <[email protected]>
1 parent 7750cf5 commit 187a6a7

File tree

9 files changed

+620
-28
lines changed

9 files changed

+620
-28
lines changed

docs/changelog/119772.yaml

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 119772
2+
summary: ESQL Support IN operator for Date nanos
3+
area: ES|QL
4+
type: enhancement
5+
issues:
6+
- 118578

x-pack/plugin/esql/build.gradle

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -283,8 +283,9 @@ tasks.named('checkstyleMain').configure {
283283
exclude { normalize(it.file.toString()).contains("src/main/generated") }
284284
}
285285

286-
def prop(Type, type, TYPE, BYTES, Array) {
286+
def prop(Name, Type, type, TYPE, BYTES, Array) {
287287
return [
288+
"Name" : Name,
288289
"Type" : Type,
289290
"type" : type,
290291
"TYPE" : TYPE,
@@ -296,15 +297,19 @@ def prop(Type, type, TYPE, BYTES, Array) {
296297
"double" : type == "double" ? "true" : "",
297298
"BytesRef" : type == "BytesRef" ? "true" : "",
298299
"boolean" : type == "boolean" ? "true" : "",
300+
"nanosMillis" : Name == "NanosMillis" ? "true" : "",
301+
"millisNanos" : Name == "MillisNanos" ? "true" : "",
299302
]
300303
}
301304

302305
tasks.named('stringTemplates').configure {
303-
var intProperties = prop("Int", "int", "INT", "Integer.BYTES", "IntArray")
304-
var longProperties = prop("Long", "long", "LONG", "Long.BYTES", "LongArray")
305-
var doubleProperties = prop("Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray")
306-
var bytesRefProperties = prop("BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "")
307-
var booleanProperties = prop("Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray")
306+
var intProperties = prop("Int", "Int", "int", "INT", "Integer.BYTES", "IntArray")
307+
var longProperties = prop("Long", "Long", "long", "LONG", "Long.BYTES", "LongArray")
308+
var nanosMillisProperties = prop("NanosMillis", "Long", "long", "LONG", "Long.BYTES", "LongArray")
309+
var millisNanosProperties = prop("MillisNanos", "Long", "long", "LONG", "Long.BYTES", "LongArray")
310+
var doubleProperties = prop("Double", "Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray")
311+
var bytesRefProperties = prop("BytesRef", "BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "")
312+
var booleanProperties = prop("Boolean", "Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray")
308313

309314
File inInputFile = file("src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st")
310315
template {
@@ -322,6 +327,16 @@ tasks.named('stringTemplates').configure {
322327
it.inputFile = inInputFile
323328
it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InLongEvaluator.java"
324329
}
330+
template {
331+
it.properties = nanosMillisProperties
332+
it.inputFile = inInputFile
333+
it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java"
334+
}
335+
template {
336+
it.properties = millisNanosProperties
337+
it.inputFile = inInputFile
338+
it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java"
339+
}
325340
template {
326341
it.properties = doubleProperties
327342
it.inputFile = inInputFile

x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1450,3 +1450,47 @@ FROM employees
14501450
cnt:long
14511451
19
14521452
;
1453+
1454+
implicit casting strings to dates for IN operator
1455+
FROM employees
1456+
| WHERE birth_date IN ("1953-04-20", "1958-10-31")
1457+
| KEEP emp_no, first_name;
1458+
ignoreOrder:true
1459+
1460+
emp_no:integer | first_name:keyword
1461+
10006 | Anneke
1462+
10025 | Prasadram
1463+
;
1464+
1465+
IN operator with null in list, finds match
1466+
1467+
FROM employees
1468+
| EVAL x = NULL
1469+
| WHERE birth_date IN (TO_DATETIME("1958-02-19T00:00:00Z"), x)
1470+
| KEEP birth_date, first_name;
1471+
1472+
birth_date:datetime | first_name:keyword
1473+
1958-02-19T00:00:00.000Z | Saniya
1474+
;
1475+
1476+
IN operator with null in list, doesn't find match
1477+
1478+
FROM employees
1479+
| EVAL x = NULL
1480+
| WHERE birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x)
1481+
| KEEP birth_date, first_name;
1482+
1483+
birth_date:datetime | first_name:keyword
1484+
;
1485+
1486+
IN operator with null in list, doesn't find match, EVAL to check value
1487+
1488+
FROM employees
1489+
| EVAL x = NULL
1490+
| EVAL result = birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x)
1491+
| LIMIT 1
1492+
| KEEP result;
1493+
1494+
result:boolean
1495+
null
1496+
;

x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1161,3 +1161,68 @@ row dt = to_date_nanos(0::long)
11611161
plus:date_nanos
11621162
1970-01-01T00:00:01.000Z
11631163
;
1164+
1165+
Date Nanos IN constant date nanos
1166+
required_capability: date_nanos_in_operator
1167+
required_capability: to_date_nanos
1168+
1169+
FROM date_nanos
1170+
| WHERE MV_FIRST(nanos) IN (TO_DATE_NANOS("2023-10-23T13:55:01.543123456Z"), TO_DATE_NANOS("2023-10-23T12:27:28.948Z"), TO_DATE_NANOS("2017-10-23T13:53:55.832987654Z"));
1171+
ignoreOrder:true
1172+
1173+
millis:date | nanos:date_nanos | num:long
1174+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
1175+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
1176+
;
1177+
1178+
Date Nanos IN constant date nanos, implicit casting
1179+
required_capability: date_nanos_in_operator
1180+
required_capability: to_date_nanos
1181+
required_capability: date_nanos_implicit_casting
1182+
1183+
FROM date_nanos
1184+
| WHERE MV_FIRST(nanos) IN ("2023-10-23T13:55:01.543123456Z", "2023-10-23T12:27:28.948Z", "2017-10-23T13:53:55.832987654Z");
1185+
ignoreOrder:true
1186+
1187+
millis:date | nanos:date_nanos | num:long
1188+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
1189+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
1190+
;
1191+
1192+
Date Nanos IN date nanos field, implicit casting
1193+
required_capability: date_nanos_in_operator
1194+
required_capability: to_date_nanos
1195+
required_capability: date_nanos_implicit_casting
1196+
1197+
FROM date_nanos
1198+
| WHERE "2023-10-23T13:55:01.543123456Z" IN (MV_FIRST(nanos));
1199+
1200+
millis:date | nanos:date_nanos | num:long
1201+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
1202+
;
1203+
1204+
Date nanos IN millisecond date field
1205+
required_capability: date_nanos_in_operator
1206+
required_capability: to_date_nanos
1207+
required_capability: date_nanos_implicit_casting
1208+
1209+
# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an ==
1210+
FROM date_nanos
1211+
| WHERE MV_FIRST(nanos) IN (TO_DATETIME("2023-10-23T12:27:28.948"), TO_DATETIME("2020-02-02"));
1212+
1213+
millis:date | nanos:date_nanos | num:long
1214+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
1215+
;
1216+
1217+
Millisecond field IN date nanos constants
1218+
required_capability: date_nanos_in_operator
1219+
required_capability: to_date_nanos
1220+
required_capability: date_nanos_implicit_casting
1221+
1222+
# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an ==
1223+
FROM date_nanos
1224+
| WHERE MV_FIRST(millis) IN (TO_DATE_NANOS("2023-10-23T12:27:28.948"), TO_DATE_NANOS("2020-02-02"));
1225+
1226+
millis:date | nanos:date_nanos | num:long
1227+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
1228+
;
Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
9+
10+
import org.elasticsearch.compute.data.Block;
11+
import org.elasticsearch.compute.data.BooleanBlock;
12+
import org.elasticsearch.compute.data.LongBlock;
13+
import org.elasticsearch.compute.data.LongVector;
14+
import org.elasticsearch.compute.data.Page;
15+
import org.elasticsearch.compute.operator.DriverContext;
16+
import org.elasticsearch.compute.operator.EvalOperator;
17+
import org.elasticsearch.compute.operator.Warnings;
18+
import org.elasticsearch.core.Releasable;
19+
import org.elasticsearch.core.Releasables;
20+
import org.elasticsearch.xpack.esql.core.tree.Source;
21+
22+
import java.util.Arrays;
23+
import java.util.BitSet;
24+
25+
/**
26+
* {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}.
27+
* This class is generated. Edit {@code X-InEvaluator.java.st} instead.
28+
*/
29+
public class InMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator {
30+
private final Source source;
31+
32+
private final EvalOperator.ExpressionEvaluator lhs;
33+
34+
private final EvalOperator.ExpressionEvaluator[] rhs;
35+
36+
private final DriverContext driverContext;
37+
38+
private Warnings warnings;
39+
40+
public InMillisNanosEvaluator(
41+
Source source,
42+
EvalOperator.ExpressionEvaluator lhs,
43+
EvalOperator.ExpressionEvaluator[] rhs,
44+
DriverContext driverContext
45+
) {
46+
this.source = source;
47+
this.lhs = lhs;
48+
this.rhs = rhs;
49+
this.driverContext = driverContext;
50+
}
51+
52+
@Override
53+
public Block eval(Page page) {
54+
try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) {
55+
LongBlock[] rhsBlocks = new LongBlock[rhs.length];
56+
try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) {
57+
for (int i = 0; i < rhsBlocks.length; i++) {
58+
rhsBlocks[i] = (LongBlock) rhs[i].eval(page);
59+
}
60+
LongVector lhsVector = lhsBlock.asVector();
61+
if (lhsVector == null) {
62+
return eval(page.getPositionCount(), lhsBlock, rhsBlocks);
63+
}
64+
LongVector[] rhsVectors = new LongVector[rhs.length];
65+
for (int i = 0; i < rhsBlocks.length; i++) {
66+
rhsVectors[i] = rhsBlocks[i].asVector();
67+
if (rhsVectors[i] == null) {
68+
return eval(page.getPositionCount(), lhsBlock, rhsBlocks);
69+
}
70+
}
71+
return eval(page.getPositionCount(), lhsVector, rhsVectors);
72+
}
73+
}
74+
}
75+
76+
private BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock[] rhsBlocks) {
77+
try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
78+
long[] rhsValues = new long[rhs.length];
79+
BitSet nulls = new BitSet(rhs.length);
80+
BitSet mvs = new BitSet(rhs.length);
81+
boolean foundMatch;
82+
for (int p = 0; p < positionCount; p++) {
83+
if (lhsBlock.isNull(p)) {
84+
result.appendNull();
85+
continue;
86+
}
87+
if (lhsBlock.getValueCount(p) != 1) {
88+
if (lhsBlock.getValueCount(p) > 1) {
89+
warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
90+
}
91+
result.appendNull();
92+
continue;
93+
}
94+
// unpack rhsBlocks into rhsValues
95+
nulls.clear();
96+
mvs.clear();
97+
for (int i = 0; i < rhsBlocks.length; i++) {
98+
if (rhsBlocks[i].isNull(p)) {
99+
nulls.set(i);
100+
continue;
101+
}
102+
if (rhsBlocks[i].getValueCount(p) > 1) {
103+
mvs.set(i);
104+
warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
105+
continue;
106+
}
107+
int o = rhsBlocks[i].getFirstValueIndex(p);
108+
rhsValues[i] = rhsBlocks[i].getLong(o);
109+
}
110+
if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) {
111+
result.appendNull();
112+
continue;
113+
}
114+
foundMatch = In.process(nulls, mvs, lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsValues);
115+
if (foundMatch) {
116+
result.appendBoolean(true);
117+
} else {
118+
if (nulls.cardinality() > 0) {
119+
result.appendNull();
120+
} else {
121+
result.appendBoolean(false);
122+
}
123+
}
124+
}
125+
return result.build();
126+
}
127+
}
128+
129+
private BooleanBlock eval(int positionCount, LongVector lhsVector, LongVector[] rhsVectors) {
130+
try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
131+
long[] rhsValues = new long[rhs.length];
132+
for (int p = 0; p < positionCount; p++) {
133+
// unpack rhsVectors into rhsValues
134+
for (int i = 0; i < rhsVectors.length; i++) {
135+
rhsValues[i] = rhsVectors[i].getLong(p);
136+
}
137+
result.appendBoolean(In.processMillisNanos(null, null, lhsVector.getLong(p), rhsValues));
138+
}
139+
return result.build();
140+
}
141+
}
142+
143+
@Override
144+
public String toString() {
145+
return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
146+
}
147+
148+
@Override
149+
public void close() {
150+
Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs));
151+
}
152+
153+
private Warnings warnings() {
154+
if (warnings == null) {
155+
this.warnings = Warnings.createWarnings(
156+
driverContext.warningsMode(),
157+
source.source().getLineNumber(),
158+
source.source().getColumnNumber(),
159+
source.text()
160+
);
161+
}
162+
return warnings;
163+
}
164+
165+
static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
166+
private final Source source;
167+
private final EvalOperator.ExpressionEvaluator.Factory lhs;
168+
private final EvalOperator.ExpressionEvaluator.Factory[] rhs;
169+
170+
Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) {
171+
this.source = source;
172+
this.lhs = lhs;
173+
this.rhs = rhs;
174+
}
175+
176+
@Override
177+
public InMillisNanosEvaluator get(DriverContext context) {
178+
EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs)
179+
.map(a -> a.get(context))
180+
.toArray(EvalOperator.ExpressionEvaluator[]::new);
181+
return new InMillisNanosEvaluator(source, lhs.get(context), rhs, context);
182+
}
183+
184+
@Override
185+
public String toString() {
186+
return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
187+
}
188+
}
189+
}

0 commit comments

Comments
 (0)