Skip to content

Commit 4f0e47c

Browse files
not-napoleonelasticsearchmachine
not-napoleon
and
elasticsearchmachine
committed
ESQL Support IN operator for Date nanos (elastic#119772)
Add support for using nanosecond dates with the IN operator. This behavior should be consistent with equals, and support comparisons between milliseconds and nanoseconds the same as the binary comparison operators support it. Resolves elastic#118578 --------- Co-authored-by: elasticsearchmachine <[email protected]>
1 parent 051305f commit 4f0e47c

File tree

9 files changed

+620
-28
lines changed

9 files changed

+620
-28
lines changed

‎docs/changelog/119772.yaml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
pr: 119772
2+
summary: ESQL Support IN operator for Date nanos
3+
area: ES|QL
4+
type: enhancement
5+
issues:
6+
- 118578

‎x-pack/plugin/esql/build.gradle‎

Lines changed: 21 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -277,8 +277,9 @@ tasks.named('checkstyleMain').configure {
277277
exclude { normalize(it.file.toString()).contains("src/main/generated") }
278278
}
279279

280-
def prop(Type, type, TYPE, BYTES, Array) {
280+
def prop(Name, Type, type, TYPE, BYTES, Array) {
281281
return [
282+
"Name" : Name,
282283
"Type" : Type,
283284
"type" : type,
284285
"TYPE" : TYPE,
@@ -290,15 +291,19 @@ def prop(Type, type, TYPE, BYTES, Array) {
290291
"double" : type == "double" ? "true" : "",
291292
"BytesRef" : type == "BytesRef" ? "true" : "",
292293
"boolean" : type == "boolean" ? "true" : "",
294+
"nanosMillis" : Name == "NanosMillis" ? "true" : "",
295+
"millisNanos" : Name == "MillisNanos" ? "true" : "",
293296
]
294297
}
295298

296299
tasks.named('stringTemplates').configure {
297-
var intProperties = prop("Int", "int", "INT", "Integer.BYTES", "IntArray")
298-
var longProperties = prop("Long", "long", "LONG", "Long.BYTES", "LongArray")
299-
var doubleProperties = prop("Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray")
300-
var bytesRefProperties = prop("BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "")
301-
var booleanProperties = prop("Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray")
300+
var intProperties = prop("Int", "Int", "int", "INT", "Integer.BYTES", "IntArray")
301+
var longProperties = prop("Long", "Long", "long", "LONG", "Long.BYTES", "LongArray")
302+
var nanosMillisProperties = prop("NanosMillis", "Long", "long", "LONG", "Long.BYTES", "LongArray")
303+
var millisNanosProperties = prop("MillisNanos", "Long", "long", "LONG", "Long.BYTES", "LongArray")
304+
var doubleProperties = prop("Double", "Double", "double", "DOUBLE", "Double.BYTES", "DoubleArray")
305+
var bytesRefProperties = prop("BytesRef", "BytesRef", "BytesRef", "BYTES_REF", "org.apache.lucene.util.RamUsageEstimator.NUM_BYTES_OBJECT_REF", "")
306+
var booleanProperties = prop("Boolean", "Boolean", "boolean", "BOOLEAN", "Byte.BYTES", "BitArray")
302307

303308
File inInputFile = file("src/main/java/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/X-InEvaluator.java.st")
304309
template {
@@ -316,6 +321,16 @@ tasks.named('stringTemplates').configure {
316321
it.inputFile = inInputFile
317322
it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InLongEvaluator.java"
318323
}
324+
template {
325+
it.properties = nanosMillisProperties
326+
it.inputFile = inInputFile
327+
it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InNanosMillisEvaluator.java"
328+
}
329+
template {
330+
it.properties = millisNanosProperties
331+
it.inputFile = inInputFile
332+
it.outputFile = "org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java"
333+
}
319334
template {
320335
it.properties = doubleProperties
321336
it.inputFile = inInputFile

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/date.csv-spec‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1451,3 +1451,47 @@ FROM employees
14511451
cnt:long
14521452
19
14531453
;
1454+
1455+
implicit casting strings to dates for IN operator
1456+
FROM employees
1457+
| WHERE birth_date IN ("1953-04-20", "1958-10-31")
1458+
| KEEP emp_no, first_name;
1459+
ignoreOrder:true
1460+
1461+
emp_no:integer | first_name:keyword
1462+
10006 | Anneke
1463+
10025 | Prasadram
1464+
;
1465+
1466+
IN operator with null in list, finds match
1467+
1468+
FROM employees
1469+
| EVAL x = NULL
1470+
| WHERE birth_date IN (TO_DATETIME("1958-02-19T00:00:00Z"), x)
1471+
| KEEP birth_date, first_name;
1472+
1473+
birth_date:datetime | first_name:keyword
1474+
1958-02-19T00:00:00.000Z | Saniya
1475+
;
1476+
1477+
IN operator with null in list, doesn't find match
1478+
1479+
FROM employees
1480+
| EVAL x = NULL
1481+
| WHERE birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x)
1482+
| KEEP birth_date, first_name;
1483+
1484+
birth_date:datetime | first_name:keyword
1485+
;
1486+
1487+
IN operator with null in list, doesn't find match, EVAL to check value
1488+
1489+
FROM employees
1490+
| EVAL x = NULL
1491+
| EVAL result = birth_date IN (TO_DATETIME("1900-02-19T00:00:00Z"), x)
1492+
| LIMIT 1
1493+
| KEEP result;
1494+
1495+
result:boolean
1496+
null
1497+
;

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/date_nanos.csv-spec‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1153,3 +1153,68 @@ row dt = to_date_nanos(0::long)
11531153
plus:date_nanos
11541154
1970-01-01T00:00:01.000Z
11551155
;
1156+
1157+
Date Nanos IN constant date nanos
1158+
required_capability: date_nanos_in_operator
1159+
required_capability: to_date_nanos
1160+
1161+
FROM date_nanos
1162+
| WHERE MV_FIRST(nanos) IN (TO_DATE_NANOS("2023-10-23T13:55:01.543123456Z"), TO_DATE_NANOS("2023-10-23T12:27:28.948Z"), TO_DATE_NANOS("2017-10-23T13:53:55.832987654Z"));
1163+
ignoreOrder:true
1164+
1165+
millis:date | nanos:date_nanos | num:long
1166+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
1167+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
1168+
;
1169+
1170+
Date Nanos IN constant date nanos, implicit casting
1171+
required_capability: date_nanos_in_operator
1172+
required_capability: to_date_nanos
1173+
required_capability: date_nanos_implicit_casting
1174+
1175+
FROM date_nanos
1176+
| WHERE MV_FIRST(nanos) IN ("2023-10-23T13:55:01.543123456Z", "2023-10-23T12:27:28.948Z", "2017-10-23T13:53:55.832987654Z");
1177+
ignoreOrder:true
1178+
1179+
millis:date | nanos:date_nanos | num:long
1180+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
1181+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
1182+
;
1183+
1184+
Date Nanos IN date nanos field, implicit casting
1185+
required_capability: date_nanos_in_operator
1186+
required_capability: to_date_nanos
1187+
required_capability: date_nanos_implicit_casting
1188+
1189+
FROM date_nanos
1190+
| WHERE "2023-10-23T13:55:01.543123456Z" IN (MV_FIRST(nanos));
1191+
1192+
millis:date | nanos:date_nanos | num:long
1193+
2023-10-23T13:55:01.543Z | 2023-10-23T13:55:01.543123456Z | 1698069301543123456
1194+
;
1195+
1196+
Date nanos IN millisecond date field
1197+
required_capability: date_nanos_in_operator
1198+
required_capability: to_date_nanos
1199+
required_capability: date_nanos_implicit_casting
1200+
1201+
# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an ==
1202+
FROM date_nanos
1203+
| WHERE MV_FIRST(nanos) IN (TO_DATETIME("2023-10-23T12:27:28.948"), TO_DATETIME("2020-02-02"));
1204+
1205+
millis:date | nanos:date_nanos | num:long
1206+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
1207+
;
1208+
1209+
Millisecond field IN date nanos constants
1210+
required_capability: date_nanos_in_operator
1211+
required_capability: to_date_nanos
1212+
required_capability: date_nanos_implicit_casting
1213+
1214+
# Note: It is important to have two dates in the IN so the optimizer doesn't turn this into an ==
1215+
FROM date_nanos
1216+
| WHERE MV_FIRST(millis) IN (TO_DATE_NANOS("2023-10-23T12:27:28.948"), TO_DATE_NANOS("2020-02-02"));
1217+
1218+
millis:date | nanos:date_nanos | num:long
1219+
2023-10-23T12:27:28.948Z | 2023-10-23T12:27:28.948000000Z | 1698064048948000000
1220+
;

‎x-pack/plugin/esql/src/main/generated-src/org/elasticsearch/xpack/esql/expression/predicate/operator/comparison/InMillisNanosEvaluator.java‎

Lines changed: 189 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,189 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.esql.expression.predicate.operator.comparison;
9+
10+
import org.elasticsearch.compute.data.Block;
11+
import org.elasticsearch.compute.data.BooleanBlock;
12+
import org.elasticsearch.compute.data.LongBlock;
13+
import org.elasticsearch.compute.data.LongVector;
14+
import org.elasticsearch.compute.data.Page;
15+
import org.elasticsearch.compute.operator.DriverContext;
16+
import org.elasticsearch.compute.operator.EvalOperator;
17+
import org.elasticsearch.compute.operator.Warnings;
18+
import org.elasticsearch.core.Releasable;
19+
import org.elasticsearch.core.Releasables;
20+
import org.elasticsearch.xpack.esql.core.tree.Source;
21+
22+
import java.util.Arrays;
23+
import java.util.BitSet;
24+
25+
/**
26+
* {@link EvalOperator.ExpressionEvaluator} implementation for {@link In}.
27+
* This class is generated. Edit {@code X-InEvaluator.java.st} instead.
28+
*/
29+
public class InMillisNanosEvaluator implements EvalOperator.ExpressionEvaluator {
30+
private final Source source;
31+
32+
private final EvalOperator.ExpressionEvaluator lhs;
33+
34+
private final EvalOperator.ExpressionEvaluator[] rhs;
35+
36+
private final DriverContext driverContext;
37+
38+
private Warnings warnings;
39+
40+
public InMillisNanosEvaluator(
41+
Source source,
42+
EvalOperator.ExpressionEvaluator lhs,
43+
EvalOperator.ExpressionEvaluator[] rhs,
44+
DriverContext driverContext
45+
) {
46+
this.source = source;
47+
this.lhs = lhs;
48+
this.rhs = rhs;
49+
this.driverContext = driverContext;
50+
}
51+
52+
@Override
53+
public Block eval(Page page) {
54+
try (LongBlock lhsBlock = (LongBlock) lhs.eval(page)) {
55+
LongBlock[] rhsBlocks = new LongBlock[rhs.length];
56+
try (Releasable rhsRelease = Releasables.wrap(rhsBlocks)) {
57+
for (int i = 0; i < rhsBlocks.length; i++) {
58+
rhsBlocks[i] = (LongBlock) rhs[i].eval(page);
59+
}
60+
LongVector lhsVector = lhsBlock.asVector();
61+
if (lhsVector == null) {
62+
return eval(page.getPositionCount(), lhsBlock, rhsBlocks);
63+
}
64+
LongVector[] rhsVectors = new LongVector[rhs.length];
65+
for (int i = 0; i < rhsBlocks.length; i++) {
66+
rhsVectors[i] = rhsBlocks[i].asVector();
67+
if (rhsVectors[i] == null) {
68+
return eval(page.getPositionCount(), lhsBlock, rhsBlocks);
69+
}
70+
}
71+
return eval(page.getPositionCount(), lhsVector, rhsVectors);
72+
}
73+
}
74+
}
75+
76+
private BooleanBlock eval(int positionCount, LongBlock lhsBlock, LongBlock[] rhsBlocks) {
77+
try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
78+
long[] rhsValues = new long[rhs.length];
79+
BitSet nulls = new BitSet(rhs.length);
80+
BitSet mvs = new BitSet(rhs.length);
81+
boolean foundMatch;
82+
for (int p = 0; p < positionCount; p++) {
83+
if (lhsBlock.isNull(p)) {
84+
result.appendNull();
85+
continue;
86+
}
87+
if (lhsBlock.getValueCount(p) != 1) {
88+
if (lhsBlock.getValueCount(p) > 1) {
89+
warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
90+
}
91+
result.appendNull();
92+
continue;
93+
}
94+
// unpack rhsBlocks into rhsValues
95+
nulls.clear();
96+
mvs.clear();
97+
for (int i = 0; i < rhsBlocks.length; i++) {
98+
if (rhsBlocks[i].isNull(p)) {
99+
nulls.set(i);
100+
continue;
101+
}
102+
if (rhsBlocks[i].getValueCount(p) > 1) {
103+
mvs.set(i);
104+
warnings().registerException(new IllegalArgumentException("single-value function encountered multi-value"));
105+
continue;
106+
}
107+
int o = rhsBlocks[i].getFirstValueIndex(p);
108+
rhsValues[i] = rhsBlocks[i].getLong(o);
109+
}
110+
if (nulls.cardinality() == rhsBlocks.length || mvs.cardinality() == rhsBlocks.length) {
111+
result.appendNull();
112+
continue;
113+
}
114+
foundMatch = In.process(nulls, mvs, lhsBlock.getLong(lhsBlock.getFirstValueIndex(p)), rhsValues);
115+
if (foundMatch) {
116+
result.appendBoolean(true);
117+
} else {
118+
if (nulls.cardinality() > 0) {
119+
result.appendNull();
120+
} else {
121+
result.appendBoolean(false);
122+
}
123+
}
124+
}
125+
return result.build();
126+
}
127+
}
128+
129+
private BooleanBlock eval(int positionCount, LongVector lhsVector, LongVector[] rhsVectors) {
130+
try (BooleanBlock.Builder result = driverContext.blockFactory().newBooleanBlockBuilder(positionCount)) {
131+
long[] rhsValues = new long[rhs.length];
132+
for (int p = 0; p < positionCount; p++) {
133+
// unpack rhsVectors into rhsValues
134+
for (int i = 0; i < rhsVectors.length; i++) {
135+
rhsValues[i] = rhsVectors[i].getLong(p);
136+
}
137+
result.appendBoolean(In.processMillisNanos(null, null, lhsVector.getLong(p), rhsValues));
138+
}
139+
return result.build();
140+
}
141+
}
142+
143+
@Override
144+
public String toString() {
145+
return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
146+
}
147+
148+
@Override
149+
public void close() {
150+
Releasables.closeExpectNoException(lhs, () -> Releasables.close(rhs));
151+
}
152+
153+
private Warnings warnings() {
154+
if (warnings == null) {
155+
this.warnings = Warnings.createWarnings(
156+
driverContext.warningsMode(),
157+
source.source().getLineNumber(),
158+
source.source().getColumnNumber(),
159+
source.text()
160+
);
161+
}
162+
return warnings;
163+
}
164+
165+
static class Factory implements EvalOperator.ExpressionEvaluator.Factory {
166+
private final Source source;
167+
private final EvalOperator.ExpressionEvaluator.Factory lhs;
168+
private final EvalOperator.ExpressionEvaluator.Factory[] rhs;
169+
170+
Factory(Source source, EvalOperator.ExpressionEvaluator.Factory lhs, EvalOperator.ExpressionEvaluator.Factory[] rhs) {
171+
this.source = source;
172+
this.lhs = lhs;
173+
this.rhs = rhs;
174+
}
175+
176+
@Override
177+
public InMillisNanosEvaluator get(DriverContext context) {
178+
EvalOperator.ExpressionEvaluator[] rhs = Arrays.stream(this.rhs)
179+
.map(a -> a.get(context))
180+
.toArray(EvalOperator.ExpressionEvaluator[]::new);
181+
return new InMillisNanosEvaluator(source, lhs.get(context), rhs, context);
182+
}
183+
184+
@Override
185+
public String toString() {
186+
return "InMillisNanosEvaluator[" + "lhs=" + lhs + ", rhs=" + Arrays.toString(rhs) + "]";
187+
}
188+
}
189+
}

0 commit comments

Comments
 (0)