Skip to content

Commit 4b96c73

Browse files
not-napoleonelasticsearchmachine
not-napoleon
and
elasticsearchmachine
authored
Esql - rework date diff tests (#120328) (#120408)
In preparation for adding nanosecond date support to date diff, this PR expands the test coverage and better integrates with the test generation framework. Using random data for date diff is extremely difficult, due to the vast range of units and the constraints of the integer return type. For example, nanosecond resolution differences of more than a couple of seconds will overflow the return value. So, instead of using a fully randomized dataset, I took the existing data set we were running against just the processing function, and formalized that to run against the whole test suite. It should be easy from here to add in cases for the same values as nanoseconds. I was able to integrate the overflow error case into the normal test framework, but the incorrect unit error doesn't play nicely with the framework (I believe this is because it fails in toEvaluator rather than in process, but I didn't dig in much). I've moved it to another test file so we don't run it for every case in the parameterized test file. --------- Co-authored-by: elasticsearchmachine <[email protected]>
1 parent 973610a commit 4b96c73

File tree

2 files changed

+155
-122
lines changed

2 files changed

+155
-122
lines changed

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffFunctionTests.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
/*
2+
* Copyright Elasticsearch B.V. and/or licensed to Elasticsearch B.V. under one
3+
* or more contributor license agreements. Licensed under the Elastic License
4+
* 2.0; you may not use this file except in compliance with the Elastic License
5+
* 2.0.
6+
*/
7+
8+
package org.elasticsearch.xpack.esql.expression.function.scalar.date;
9+
10+
import org.apache.lucene.util.BytesRef;
11+
import org.elasticsearch.test.ESTestCase;
12+
13+
import static org.hamcrest.Matchers.containsString;
14+
15+
/**
16+
* Tests for {@link DateDiff} that should not run through the normal testing framework
17+
*/
18+
public class DateDiffFunctionTests extends ESTestCase {
19+
20+
public void testDateDiffFunctionErrorUnitNotValid() {
21+
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> DateDiff.process(new BytesRef("sseconds"), 0, 0));
22+
assertThat(
23+
e.getMessage(),
24+
containsString(
25+
"Received value [sseconds] is not valid date part to add; "
26+
+ "did you mean [seconds, second, nanoseconds, milliseconds, microseconds, nanosecond]?"
27+
)
28+
);
29+
30+
e = expectThrows(IllegalArgumentException.class, () -> DateDiff.process(new BytesRef("not-valid-unit"), 0, 0));
31+
assertThat(
32+
e.getMessage(),
33+
containsString(
34+
"A value of [YEAR, QUARTER, MONTH, DAYOFYEAR, DAY, WEEK, WEEKDAY, HOUR, MINUTE, SECOND, MILLISECOND, MICROSECOND, "
35+
+ "NANOSECOND] or their aliases is required; received [not-valid-unit]"
36+
)
37+
);
38+
}
39+
40+
}

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/expression/function/scalar/date/DateDiffTests.java‎

Lines changed: 115 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -11,19 +11,17 @@
1111
import com.carrotsearch.randomizedtesting.annotations.ParametersFactory;
1212

1313
import org.apache.lucene.util.BytesRef;
14-
import org.elasticsearch.xpack.esql.core.InvalidArgumentException;
1514
import org.elasticsearch.xpack.esql.core.expression.Expression;
1615
import org.elasticsearch.xpack.esql.core.tree.Source;
1716
import org.elasticsearch.xpack.esql.core.type.DataType;
1817
import org.elasticsearch.xpack.esql.expression.function.AbstractScalarFunctionTestCase;
1918
import org.elasticsearch.xpack.esql.expression.function.TestCaseSupplier;
2019

21-
import java.time.ZonedDateTime;
20+
import java.time.Instant;
2221
import java.util.ArrayList;
2322
import java.util.List;
2423
import java.util.function.Supplier;
2524

26-
import static org.hamcrest.Matchers.containsString;
2725
import static org.hamcrest.Matchers.equalTo;
2826

2927
public class DateDiffTests extends AbstractScalarFunctionTestCase {
@@ -33,149 +31,144 @@ public DateDiffTests(@Name("TestCase") Supplier<TestCaseSupplier.TestCase> testC
3331

3432
@ParametersFactory
3533
public static Iterable<Object[]> parameters() {
36-
ZonedDateTime zdtStart = ZonedDateTime.parse("2023-12-04T10:15:30Z");
37-
ZonedDateTime zdtEnd = ZonedDateTime.parse("2023-12-05T10:45:00Z");
3834

3935
List<TestCaseSupplier> suppliers = new ArrayList<>();
40-
suppliers.add(
36+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-04T10:15:30Z"), Instant.parse("2023-12-05T10:45:00Z"), "seconds", 88170));
37+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-12T00:01:01Z"), Instant.parse("2024-12-12T00:01:01Z"), "year", 1));
38+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-12T00:01:01.001Z"), Instant.parse("2024-12-12T00:01:01Z"), "year", 0));
39+
40+
suppliers.addAll(
41+
makeSuppliers(Instant.parse("2023-12-04T10:15:00Z"), Instant.parse("2023-12-04T10:15:01Z"), "nanoseconds", 1000000000)
42+
);
43+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-04T10:15:00Z"), Instant.parse("2023-12-04T10:15:01Z"), "ns", 1000000000));
44+
suppliers.addAll(
45+
makeSuppliers(Instant.parse("2023-12-04T10:15:00Z"), Instant.parse("2023-12-04T10:15:01Z"), "microseconds", 1000000)
46+
);
47+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-04T10:15:00Z"), Instant.parse("2023-12-04T10:15:01Z"), "mcs", 1000000));
48+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-04T10:15:00Z"), Instant.parse("2023-12-04T10:15:01Z"), "milliseconds", 1000));
49+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-04T10:15:00Z"), Instant.parse("2023-12-04T10:15:01Z"), "ms", 1000));
50+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-04T10:15:00Z"), Instant.parse("2023-12-04T10:15:01Z"), "seconds", 1));
51+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-04T10:15:00Z"), Instant.parse("2023-12-04T10:15:01Z"), "ss", 1));
52+
suppliers.addAll(makeSuppliers(Instant.parse("2023-12-04T10:15:00Z"), Instant.parse("2023-12-04T10:15:01Z"), "s", 1));
53+
54+
Instant zdtStart = Instant.parse("2023-12-04T10:15:00Z");
55+
Instant zdtEnd = Instant.parse("2024-12-04T10:15:01Z");
56+
57+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "minutes", 527040));
58+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "mi", 527040));
59+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "n", 527040));
60+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "hours", 8784));
61+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "hh", 8784));
62+
63+
// 2024 is a leap year, so the dates are 366 days apart
64+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "weekdays", 366));
65+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "dw", 366));
66+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "days", 366));
67+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "dd", 366));
68+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "d", 366));
69+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "dy", 366));
70+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "y", 366));
71+
72+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "weeks", 52));
73+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "wk", 52));
74+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "ww", 52));
75+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "months", 12));
76+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "mm", 12));
77+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "m", 12));
78+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "quarters", 4));
79+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "qq", 4));
80+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "q", 4));
81+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "years", 1));
82+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "yyyy", 1));
83+
suppliers.addAll(makeSuppliers(zdtStart, zdtEnd, "yy", 1));
84+
85+
// Error cases
86+
Instant zdtStart2 = Instant.parse("2023-12-04T10:15:00Z");
87+
Instant zdtEnd2 = Instant.parse("2023-12-04T10:20:00Z");
88+
suppliers.addAll(
89+
makeSuppliers(
90+
zdtStart2,
91+
zdtEnd2,
92+
"nanoseconds",
93+
"Line -1:-1: org.elasticsearch.xpack.esql.core.InvalidArgumentException: [300000000000] out of [integer] range"
94+
)
95+
);
96+
97+
return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
98+
}
99+
100+
private static List<TestCaseSupplier> makeSuppliers(Instant startTimestamp, Instant endTimestamp, String unit, int expected) {
101+
// Units as Keyword case
102+
return List.of(
41103
new TestCaseSupplier(
42-
"Date Diff In Seconds - OK",
104+
"DateDiff(" + unit + "<KEYWORD>, " + startTimestamp + ", " + endTimestamp + ") == " + expected,
43105
List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME),
44106
() -> new TestCaseSupplier.TestCase(
45107
List.of(
46-
new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.KEYWORD, "unit"),
47-
new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
48-
new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
108+
new TestCaseSupplier.TypedData(new BytesRef(unit), DataType.KEYWORD, "unit"),
109+
new TestCaseSupplier.TypedData(startTimestamp.toEpochMilli(), DataType.DATETIME, "startTimestamp"),
110+
new TestCaseSupplier.TypedData(endTimestamp.toEpochMilli(), DataType.DATETIME, "endTimestamp")
49111
),
50112
"DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
51113
+ "endTimestamp=Attribute[channel=2]]",
52114
DataType.INTEGER,
53-
equalTo(88170)
115+
equalTo(expected)
54116
)
55-
)
56-
);
57-
suppliers.add(
117+
),
118+
// Units as text case
58119
new TestCaseSupplier(
59-
"Date Diff In Seconds with text- OK",
120+
"DateDiff(" + unit + "<TEXT>, " + startTimestamp + ", " + endTimestamp + ") == " + expected,
60121
List.of(DataType.TEXT, DataType.DATETIME, DataType.DATETIME),
61122
() -> new TestCaseSupplier.TestCase(
62123
List.of(
63-
new TestCaseSupplier.TypedData(new BytesRef("seconds"), DataType.TEXT, "unit"),
64-
new TestCaseSupplier.TypedData(zdtStart.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
65-
new TestCaseSupplier.TypedData(zdtEnd.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
124+
new TestCaseSupplier.TypedData(new BytesRef(unit), DataType.TEXT, "unit"),
125+
new TestCaseSupplier.TypedData(startTimestamp.toEpochMilli(), DataType.DATETIME, "startTimestamp"),
126+
new TestCaseSupplier.TypedData(endTimestamp.toEpochMilli(), DataType.DATETIME, "endTimestamp")
66127
),
67128
"DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
68129
+ "endTimestamp=Attribute[channel=2]]",
69130
DataType.INTEGER,
70-
equalTo(88170)
131+
equalTo(expected)
71132
)
72133
)
73134
);
74-
suppliers.add(new TestCaseSupplier("Date Diff In Year - 1", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> {
75-
ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01Z");
76-
ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z");
77-
return new TestCaseSupplier.TestCase(
78-
List.of(
79-
new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"),
80-
new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
81-
new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
82-
),
83-
"DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]",
84-
DataType.INTEGER,
85-
equalTo(1)
86-
);
87-
}));
88-
suppliers.add(new TestCaseSupplier("Date Diff In Year - 0", List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME), () -> {
89-
ZonedDateTime zdtStart2 = ZonedDateTime.parse("2023-12-12T00:01:01.001Z");
90-
ZonedDateTime zdtEnd2 = ZonedDateTime.parse("2024-12-12T00:01:01Z");
91-
return new TestCaseSupplier.TestCase(
92-
List.of(
93-
new TestCaseSupplier.TypedData(new BytesRef("year"), DataType.KEYWORD, "unit"),
94-
new TestCaseSupplier.TypedData(zdtStart2.toInstant().toEpochMilli(), DataType.DATETIME, "startTimestamp"),
95-
new TestCaseSupplier.TypedData(zdtEnd2.toInstant().toEpochMilli(), DataType.DATETIME, "endTimestamp")
96-
),
97-
"DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], " + "endTimestamp=Attribute[channel=2]]",
98-
DataType.INTEGER,
99-
equalTo(0)
100-
);
101-
}));
102-
return parameterSuppliersFromTypedDataWithDefaultChecksNoErrors(true, suppliers);
103135
}
104136

105-
public void testDateDiffFunction() {
106-
ZonedDateTime zdtStart = ZonedDateTime.parse("2023-12-04T10:15:00Z");
107-
ZonedDateTime zdtEnd = ZonedDateTime.parse("2023-12-04T10:15:01Z");
108-
long startTimestamp = zdtStart.toInstant().toEpochMilli();
109-
long endTimestamp = zdtEnd.toInstant().toEpochMilli();
110-
111-
assertEquals(1000000000, DateDiff.process(new BytesRef("nanoseconds"), startTimestamp, endTimestamp));
112-
assertEquals(1000000000, DateDiff.process(new BytesRef("ns"), startTimestamp, endTimestamp));
113-
assertEquals(1000000, DateDiff.process(new BytesRef("microseconds"), startTimestamp, endTimestamp));
114-
assertEquals(1000000, DateDiff.process(new BytesRef("mcs"), startTimestamp, endTimestamp));
115-
assertEquals(1000, DateDiff.process(new BytesRef("milliseconds"), startTimestamp, endTimestamp));
116-
assertEquals(1000, DateDiff.process(new BytesRef("ms"), startTimestamp, endTimestamp));
117-
assertEquals(1, DateDiff.process(new BytesRef("seconds"), startTimestamp, endTimestamp));
118-
assertEquals(1, DateDiff.process(new BytesRef("ss"), startTimestamp, endTimestamp));
119-
assertEquals(1, DateDiff.process(new BytesRef("s"), startTimestamp, endTimestamp));
120-
121-
zdtEnd = zdtEnd.plusYears(1);
122-
endTimestamp = zdtEnd.toInstant().toEpochMilli();
123-
124-
assertEquals(527040, DateDiff.process(new BytesRef("minutes"), startTimestamp, endTimestamp));
125-
assertEquals(527040, DateDiff.process(new BytesRef("mi"), startTimestamp, endTimestamp));
126-
assertEquals(527040, DateDiff.process(new BytesRef("n"), startTimestamp, endTimestamp));
127-
assertEquals(8784, DateDiff.process(new BytesRef("hours"), startTimestamp, endTimestamp));
128-
assertEquals(8784, DateDiff.process(new BytesRef("hh"), startTimestamp, endTimestamp));
129-
assertEquals(366, DateDiff.process(new BytesRef("weekdays"), startTimestamp, endTimestamp));
130-
assertEquals(366, DateDiff.process(new BytesRef("dw"), startTimestamp, endTimestamp));
131-
assertEquals(52, DateDiff.process(new BytesRef("weeks"), startTimestamp, endTimestamp));
132-
assertEquals(52, DateDiff.process(new BytesRef("wk"), startTimestamp, endTimestamp));
133-
assertEquals(52, DateDiff.process(new BytesRef("ww"), startTimestamp, endTimestamp));
134-
assertEquals(366, DateDiff.process(new BytesRef("days"), startTimestamp, endTimestamp));
135-
assertEquals(366, DateDiff.process(new BytesRef("dd"), startTimestamp, endTimestamp));
136-
assertEquals(366, DateDiff.process(new BytesRef("d"), startTimestamp, endTimestamp));
137-
assertEquals(366, DateDiff.process(new BytesRef("dy"), startTimestamp, endTimestamp));
138-
assertEquals(366, DateDiff.process(new BytesRef("y"), startTimestamp, endTimestamp));
139-
assertEquals(12, DateDiff.process(new BytesRef("months"), startTimestamp, endTimestamp));
140-
assertEquals(12, DateDiff.process(new BytesRef("mm"), startTimestamp, endTimestamp));
141-
assertEquals(12, DateDiff.process(new BytesRef("m"), startTimestamp, endTimestamp));
142-
assertEquals(4, DateDiff.process(new BytesRef("quarters"), startTimestamp, endTimestamp));
143-
assertEquals(4, DateDiff.process(new BytesRef("qq"), startTimestamp, endTimestamp));
144-
assertEquals(4, DateDiff.process(new BytesRef("q"), startTimestamp, endTimestamp));
145-
assertEquals(1, DateDiff.process(new BytesRef("years"), startTimestamp, endTimestamp));
146-
assertEquals(1, DateDiff.process(new BytesRef("yyyy"), startTimestamp, endTimestamp));
147-
assertEquals(1, DateDiff.process(new BytesRef("yy"), startTimestamp, endTimestamp));
148-
}
149-
150-
public void testDateDiffFunctionErrorTooLarge() {
151-
ZonedDateTime zdtStart = ZonedDateTime.parse("2023-12-04T10:15:00Z");
152-
ZonedDateTime zdtEnd = ZonedDateTime.parse("2023-12-04T10:20:00Z");
153-
long startTimestamp = zdtStart.toInstant().toEpochMilli();
154-
long endTimestamp = zdtEnd.toInstant().toEpochMilli();
155-
156-
InvalidArgumentException e = expectThrows(
157-
InvalidArgumentException.class,
158-
() -> DateDiff.process(new BytesRef("nanoseconds"), startTimestamp, endTimestamp)
159-
);
160-
assertThat(e.getMessage(), containsString("[300000000000] out of [integer] range"));
161-
}
162-
163-
public void testDateDiffFunctionErrorUnitNotValid() {
164-
IllegalArgumentException e = expectThrows(IllegalArgumentException.class, () -> DateDiff.process(new BytesRef("sseconds"), 0, 0));
165-
assertThat(
166-
e.getMessage(),
167-
containsString(
168-
"Received value [sseconds] is not valid date part to add; "
169-
+ "did you mean [seconds, second, nanoseconds, milliseconds, microseconds, nanosecond]?"
170-
)
171-
);
172-
173-
e = expectThrows(IllegalArgumentException.class, () -> DateDiff.process(new BytesRef("not-valid-unit"), 0, 0));
174-
assertThat(
175-
e.getMessage(),
176-
containsString(
177-
"A value of [YEAR, QUARTER, MONTH, DAYOFYEAR, DAY, WEEK, WEEKDAY, HOUR, MINUTE, SECOND, MILLISECOND, MICROSECOND, "
178-
+ "NANOSECOND] or their aliases is required; received [not-valid-unit]"
137+
private static List<TestCaseSupplier> makeSuppliers(Instant startTimestamp, Instant endTimestamp, String unit, String warning) {
138+
// Units as Keyword case
139+
return List.of(
140+
new TestCaseSupplier(
141+
"DateDiff(" + unit + "<KEYWORD>, " + startTimestamp + ", " + endTimestamp + ") -> warning ",
142+
List.of(DataType.KEYWORD, DataType.DATETIME, DataType.DATETIME),
143+
() -> new TestCaseSupplier.TestCase(
144+
List.of(
145+
new TestCaseSupplier.TypedData(new BytesRef(unit), DataType.KEYWORD, "unit"),
146+
new TestCaseSupplier.TypedData(startTimestamp.toEpochMilli(), DataType.DATETIME, "startTimestamp"),
147+
new TestCaseSupplier.TypedData(endTimestamp.toEpochMilli(), DataType.DATETIME, "endTimestamp")
148+
),
149+
"DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
150+
+ "endTimestamp=Attribute[channel=2]]",
151+
DataType.INTEGER,
152+
equalTo(null)
153+
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
154+
.withWarning(warning)
155+
),
156+
// Units as text case
157+
new TestCaseSupplier(
158+
"DateDiff(" + unit + "<TEXT>, " + startTimestamp + ", " + endTimestamp + ") -> warning ",
159+
List.of(DataType.TEXT, DataType.DATETIME, DataType.DATETIME),
160+
() -> new TestCaseSupplier.TestCase(
161+
List.of(
162+
new TestCaseSupplier.TypedData(new BytesRef(unit), DataType.TEXT, "unit"),
163+
new TestCaseSupplier.TypedData(startTimestamp.toEpochMilli(), DataType.DATETIME, "startTimestamp"),
164+
new TestCaseSupplier.TypedData(endTimestamp.toEpochMilli(), DataType.DATETIME, "endTimestamp")
165+
),
166+
"DateDiffEvaluator[unit=Attribute[channel=0], startTimestamp=Attribute[channel=1], "
167+
+ "endTimestamp=Attribute[channel=2]]",
168+
DataType.INTEGER,
169+
equalTo(null)
170+
).withWarning("Line -1:-1: evaluation of [] failed, treating result as null. Only first 20 failures recorded.")
171+
.withWarning(warning)
179172
)
180173
);
181174
}

0 commit comments

Comments
 (0)