Skip to content

Commit 3ce645d

Browse files
author
elasticsearchmachine
committed
Merge remote-tracking branch 'origin/main' into enhancement/esql-profile-planning-time
# Conflicts: # x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/planner/PlannerUtils.java # x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/session/EsqlSession.java
2 parents 2f8bb1c + 683df11 commit 3ce645d

File tree

23 files changed

+662
-246
lines changed

23 files changed

+662
-246
lines changed

‎docs/changelog/138023.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 138023
2+
summary: Push down COUNT(*) BY DATE_TRUNC
3+
area: ES|QL
4+
type: feature
5+
issues: []

‎server/src/test/java/org/elasticsearch/action/admin/cluster/stats/MappingStatsTests.java‎

Lines changed: 21 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -99,7 +99,7 @@ public class MappingStatsTests extends AbstractWireSerializingTestCase<MappingSt
9999

100100
private static final String SCRIPT_1 = scriptAsJSON("doc['field'] + doc.field + params._source.field");
101101
private static final String SCRIPT_2 = scriptAsJSON("doc['field']");
102-
private static final String SCRIPT_3 = scriptAsJSON("params._source.field + params._source.field + params._source.field");
102+
private static final String SCRIPT_3 = scriptAsJSON("params._source.field + params._source.field \n + params._source.field");
103103
private static final String SCRIPT_4 = scriptAsJSON("params._source.field");
104104

105105
public void testToXContent() {
@@ -114,8 +114,8 @@ public void testToXContent() {
114114
"mappings" : {
115115
"total_field_count" : 12,
116116
"total_deduplicated_field_count" : 6,
117-
"total_deduplicated_mapping_size" : "255b",
118-
"total_deduplicated_mapping_size_in_bytes" : 255,
117+
"total_deduplicated_mapping_size" : "260b",
118+
"total_deduplicated_mapping_size_in_bytes" : 260,
119119
"field_types" : [
120120
{
121121
"name" : "dense_vector",
@@ -159,10 +159,10 @@ public void testToXContent() {
159159
"lang" : [
160160
"painless"
161161
],
162-
"lines_max" : 1,
163-
"lines_total" : 4,
164-
"chars_max" : 66,
165-
"chars_total" : 172,
162+
"lines_max" : 2,
163+
"lines_total" : 6,
164+
"chars_max" : 68,
165+
"chars_total" : 176,
166166
"source_max" : 3,
167167
"source_total" : 8,
168168
"doc_max" : 0,
@@ -203,10 +203,10 @@ public void testToXContent() {
203203
"lang" : [
204204
"painless"
205205
],
206-
"lines_max" : 1,
207-
"lines_total" : 4,
208-
"chars_max" : 66,
209-
"chars_total" : 172,
206+
"lines_max" : 2,
207+
"lines_total" : 6,
208+
"chars_max" : 68,
209+
"chars_total" : 176,
210210
"source_max" : 3,
211211
"source_total" : 8,
212212
"doc_max" : 0,
@@ -246,8 +246,8 @@ public void testToXContentWithSomeSharedMappings() {
246246
"mappings" : {
247247
"total_field_count" : 18,
248248
"total_deduplicated_field_count" : 12,
249-
"total_deduplicated_mapping_size" : "513b",
250-
"total_deduplicated_mapping_size_in_bytes" : 513,
249+
"total_deduplicated_mapping_size" : "519b",
250+
"total_deduplicated_mapping_size_in_bytes" : 519,
251251
"field_types" : [
252252
{
253253
"name" : "dense_vector",
@@ -291,10 +291,10 @@ public void testToXContentWithSomeSharedMappings() {
291291
"lang" : [
292292
"painless"
293293
],
294-
"lines_max" : 1,
295-
"lines_total" : 6,
296-
"chars_max" : 66,
297-
"chars_total" : 258,
294+
"lines_max" : 2,
295+
"lines_total" : 9,
296+
"chars_max" : 68,
297+
"chars_total" : 264,
298298
"source_max" : 3,
299299
"source_total" : 12,
300300
"doc_max" : 0,
@@ -335,10 +335,10 @@ public void testToXContentWithSomeSharedMappings() {
335335
"lang" : [
336336
"painless"
337337
],
338-
"lines_max" : 1,
339-
"lines_total" : 6,
340-
"chars_max" : 66,
341-
"chars_total" : 258,
338+
"lines_max" : 2,
339+
"lines_total" : 9,
340+
"chars_max" : 68,
341+
"chars_total" : 264,
342342
"source_max" : 3,
343343
"source_total" : 12,
344344
"doc_max" : 0,

‎test/framework/src/main/java/org/elasticsearch/common/bytes/AbstractBytesReferenceTestCase.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
import java.util.List;
3434
import java.util.Map;
3535

36+
import static org.hamcrest.Matchers.greaterThan;
37+
3638
public abstract class AbstractBytesReferenceTestCase extends ESTestCase {
3739

3840
protected static final int PAGE_SIZE = PageCacheRecycler.BYTE_PAGE_SIZE;
@@ -376,6 +378,7 @@ public void testIterator() throws IOException {
376378
BytesRef ref;
377379
BytesRefBuilder builder = new BytesRefBuilder();
378380
while ((ref = iterator.next()) != null) {
381+
assertThat(ref.length, greaterThan(0));
379382
builder.append(ref);
380383
}
381384
assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
@@ -388,9 +391,10 @@ public void testSliceIterator() throws IOException {
388391
int sliceLength = randomIntBetween(0, pbr.length() - sliceOffset);
389392
BytesReference slice = pbr.slice(sliceOffset, sliceLength);
390393
BytesRefIterator iterator = slice.iterator();
391-
BytesRef ref = null;
394+
BytesRef ref;
392395
BytesRefBuilder builder = new BytesRefBuilder();
393396
while ((ref = iterator.next()) != null) {
397+
assertThat(ref.length, greaterThan(0));
394398
builder.append(ref);
395399
}
396400
assertArrayEquals(BytesReference.toBytes(slice), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
@@ -409,9 +413,10 @@ public void testIteratorRandom() throws IOException {
409413
pbr = new BytesArray(pbr.toBytesRef());
410414
}
411415
BytesRefIterator iterator = pbr.iterator();
412-
BytesRef ref = null;
416+
BytesRef ref;
413417
BytesRefBuilder builder = new BytesRefBuilder();
414418
while ((ref = iterator.next()) != null) {
419+
assertThat(ref.length, greaterThan(0));
415420
builder.append(ref);
416421
}
417422
assertArrayEquals(BytesReference.toBytes(pbr), BytesRef.deepCopyOf(builder.toBytesRef()).bytes);
@@ -683,6 +688,7 @@ public void testWriteWithIterator() throws IOException {
683688
BytesRef bytesRef;
684689
int offset = 0;
685690
while ((bytesRef = iterator.next()) != null) {
691+
assertThat(bytesRef.length, greaterThan(0));
686692
final int len = Math.min(bytesRef.length, length - offset);
687693
System.arraycopy(bytes, offset, bytesRef.bytes, bytesRef.offset, len);
688694
offset += len;

‎x-pack/plugin/esql/compute/src/main/java/org/elasticsearch/compute/lucene/LuceneCountOperator.java‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -175,11 +175,11 @@ private Page buildConstantBlocksResult(List<Object> tags, PerTagsState state) {
175175
Block[] blocks = new Block[2 + tagTypes.size()];
176176
int b = 0;
177177
try {
178-
blocks[b++] = blockFactory.newConstantLongBlockWith(state.totalHits, 1);
179-
blocks[b++] = blockFactory.newConstantBooleanBlockWith(true, 1);
180-
for (Object e : tags) {
178+
for (Object e : tags) { // by
181179
blocks[b++] = BlockUtils.constantBlock(blockFactory, e, 1);
182180
}
181+
blocks[b++] = blockFactory.newConstantLongBlockWith(state.totalHits, 1); // count
182+
blocks[b] = blockFactory.newConstantBooleanBlockWith(true, 1); // seen
183183
Page page = new Page(1, blocks);
184184
blocks = null;
185185
return page;
@@ -207,12 +207,12 @@ private Page buildNonConstantBlocksResult() {
207207
}
208208
}
209209

210-
blocks[0] = countBuilder.build().asBlock();
211-
blocks[1] = blockFactory.newConstantBooleanBlockWith(true, tagsToState.size());
212-
for (b = 0; b < builders.length; b++) {
213-
blocks[2 + b] = builders[b].builder().build();
214-
builders[b++] = null;
210+
for (b = 0; b < builders.length; b++) { // by
211+
blocks[b] = builders[b].builder().build();
212+
builders[b] = null;
215213
}
214+
blocks[b++] = countBuilder.build().asBlock(); // count
215+
blocks[b] = blockFactory.newConstantBooleanBlockWith(true, tagsToState.size()); // seen
216216
Page page = new Page(tagsToState.size(), blocks);
217217
blocks = null;
218218
return page;

‎x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/OperatorTests.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -404,9 +404,9 @@ public void testPushRoundToCountToQuery() throws IOException {
404404
try (CannedSourceOperator sourceOperator = new CannedSourceOperator(dataDriverPages.iterator())) {
405405
HashAggregationOperator.HashAggregationOperatorFactory aggFactory =
406406
new HashAggregationOperator.HashAggregationOperatorFactory(
407-
List.of(new BlockHash.GroupSpec(2, ElementType.LONG)),
408-
AggregatorMode.FINAL,
409-
List.of(CountAggregatorFunction.supplier().groupingAggregatorFactory(AggregatorMode.FINAL, List.of(0, 1))),
407+
List.of(new BlockHash.GroupSpec(0, ElementType.LONG)),
408+
AggregatorMode.INTERMEDIATE,
409+
List.of(CountAggregatorFunction.supplier().groupingAggregatorFactory(AggregatorMode.INTERMEDIATE, List.of(1, 2))),
410410
Integer.MAX_VALUE,
411411
null
412412
);
@@ -426,7 +426,7 @@ public void testPushRoundToCountToQuery() throws IOException {
426426

427427
assertThat(reduceDriverPages, hasSize(1));
428428
Page result = reduceDriverPages.getFirst();
429-
assertThat(result.getBlockCount(), equalTo(2));
429+
assertThat(result.getBlockCount(), equalTo(3));
430430
LongBlock groupsBlock = result.getBlock(0);
431431
LongVector groups = groupsBlock.asVector();
432432
LongBlock countsBlock = result.getBlock(1);

‎x-pack/plugin/esql/compute/src/test/java/org/elasticsearch/compute/lucene/LuceneCountOperatorTests.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -325,7 +325,7 @@ private static long getCount(Page p) {
325325
}
326326

327327
private static void checkSeen(Page p, Matcher<Integer> positionCount) {
328-
BooleanBlock b = p.getBlock(1);
328+
BooleanBlock b = p.getBlock(p.getBlockCount() - 1);
329329
BooleanVector v = b.asVector();
330330
assertThat(v.getPositionCount(), positionCount);
331331
assertThat(v.isConstant(), equalTo(true));
@@ -337,9 +337,9 @@ private static Map<Integer, Long> getCountsByTag(List<Page> results) {
337337
for (Page page : results) {
338338
assertThat(page.getBlockCount(), equalTo(3));
339339
checkSeen(page, greaterThanOrEqualTo(0));
340-
LongBlock countsBlock = page.getBlock(0);
340+
LongBlock countsBlock = page.getBlock(page.getBlockCount() - 2);
341341
LongVector counts = countsBlock.asVector();
342-
IntBlock groupsBlock = page.getBlock(2);
342+
IntBlock groupsBlock = page.getBlock(0);
343343
IntVector groups = groupsBlock.asVector();
344344
for (int p = 0; p < page.getPositionCount(); p++) {
345345
long count = counts.getLong(p);

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/fork.csv-spec‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -179,6 +179,32 @@ fork3 | null | 100 | 10100 | 10001
179179
fork4 | null | 100 | 10001 | null
180180
;
181181

182+
forkWithStatsCountStarDateTrunc
183+
required_capability: fork_v9
184+
185+
FROM employees
186+
| FORK (WHERE emp_no == 10048 OR emp_no == 10081)
187+
(WHERE emp_no == 10081 OR emp_no == 10087)
188+
(STATS x = COUNT(*), y = MAX(emp_no) by hd = DATE_TRUNC(1 year, hire_date))
189+
(STATS x = COUNT(*), y = MIN(emp_no) by hd = DATE_TRUNC(2 year, hire_date))
190+
| KEEP _fork, emp_no, hd, x, y
191+
| SORT _fork, emp_no, hd
192+
| LIMIT 10
193+
;
194+
195+
_fork:keyword | emp_no:integer | hd:datetime | x:long | y:integer
196+
fork1 | 10048 | null | null | null
197+
fork1 | 10081 | null | null | null
198+
fork2 | 10081 | null | null | null
199+
fork2 | 10087 | null | null | null
200+
fork3 | null | 1985-01-01T00:00:00.000Z | 11 | 10098
201+
fork3 | null | 1986-01-01T00:00:00.000Z | 11 | 10095
202+
fork3 | null | 1987-01-01T00:00:00.000Z | 15 | 10100
203+
fork3 | null | 1988-01-01T00:00:00.000Z | 9 | 10099
204+
fork3 | null | 1989-01-01T00:00:00.000Z | 13 | 10092
205+
fork3 | null | 1990-01-01T00:00:00.000Z | 12 | 10097
206+
;
207+
182208
forkWithDissect
183209
required_capability: fork_v9
184210

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/inlinestats.csv-spec‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3789,6 +3789,29 @@ null |1995-08-22T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
37893789
null |1995-08-22T00:00:00.000Z|10 |1995-01-01T00:00:00.000Z
37903790
;
37913791

3792+
DateTruncInlinestatsCountStar
3793+
required_capability: inline_stats
3794+
3795+
FROM employees
3796+
| KEEP emp_no, hire_date
3797+
| INLINE STATS c = count(*) by yr=date_trunc(1 year, hire_date)
3798+
| SORT hire_date DESC
3799+
| LIMIT 10
3800+
;
3801+
3802+
emp_no:integer | hire_date:datetime |c:long | yr:datetime
3803+
10019 | 1999-04-30T00:00:00.000Z | 1 | 1999-01-01T00:00:00.000Z
3804+
10024 | 1997-05-19T00:00:00.000Z | 1 | 1997-01-01T00:00:00.000Z
3805+
10093 | 1996-11-05T00:00:00.000Z | 1 | 1996-01-01T00:00:00.000Z
3806+
10084 | 1995-12-15T00:00:00.000Z | 5 | 1995-01-01T00:00:00.000Z
3807+
10022 | 1995-08-22T00:00:00.000Z | 5 | 1995-01-01T00:00:00.000Z
3808+
10026 | 1995-03-20T00:00:00.000Z | 5 | 1995-01-01T00:00:00.000Z
3809+
10054 | 1995-03-13T00:00:00.000Z | 5 | 1995-01-01T00:00:00.000Z
3810+
10016 | 1995-01-27T00:00:00.000Z | 5 | 1995-01-01T00:00:00.000Z
3811+
10008 | 1994-09-15T00:00:00.000Z | 4 | 1994-01-01T00:00:00.000Z
3812+
10044 | 1994-05-21T00:00:00.000Z | 4 | 1994-01-01T00:00:00.000Z
3813+
;
3814+
37923815
ImplicitCastingMultiTypedBucketDateNanosByYear
37933816
required_capability: inline_stats
37943817

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/stats.csv-spec‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -835,6 +835,21 @@ c:l | l:i
835835
15 |1
836836
;
837837

838+
countStarGroupedTrunc
839+
from employees | stats c = count(*) by d=date_trunc(1 year, hire_date) | sort d | limit 10;
840+
c:l | d:datetime
841+
11 | 1985-01-01T00:00:00.000Z
842+
11 | 1986-01-01T00:00:00.000Z
843+
15 | 1987-01-01T00:00:00.000Z
844+
9 | 1988-01-01T00:00:00.000Z
845+
13 | 1989-01-01T00:00:00.000Z
846+
12 | 1990-01-01T00:00:00.000Z
847+
6 | 1991-01-01T00:00:00.000Z
848+
8 | 1992-01-01T00:00:00.000Z
849+
3 | 1993-01-01T00:00:00.000Z
850+
4 | 1994-01-01T00:00:00.000Z
851+
;
852+
838853
countAllAndOtherStatGrouped
839854
from employees | stats c = count(*), min = min(emp_no) by languages | sort languages;
840855

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/subquery.csv-spec‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -340,6 +340,28 @@ employees | 10093 | 3 | null | null
340340
null | null | null | 4 | 172.21.3.15
341341
;
342342

343+
subqueryWithCountStarAndDateTrunc
344+
required_capability: fork_v9
345+
required_capability: subquery_in_from_command
346+
347+
FROM employees, (FROM sample_data
348+
| STATS cnt = count(*) by ts=date_trunc(1 hour, @timestamp)
349+
| SORT cnt DESC
350+
)
351+
metadata _index
352+
| WHERE ( emp_no >= 10091 AND emp_no < 10094) OR emp_no IS NULL
353+
| SORT emp_no, ts
354+
| KEEP _index, emp_no, languages, cnt, ts
355+
;
356+
357+
_index:keyword | emp_no:integer | languages:integer | cnt:long | ts:datetime
358+
employees | 10091 | 3 | null | null
359+
employees | 10092 | 1 | null | null
360+
employees | 10093 | 3 | null | null
361+
null | null | null | 2 | 2023-10-23T12:00:00.000Z
362+
null | null | null | 5 | 2023-10-23T13:00:00.000Z
363+
;
364+
343365
subqueryInFromWithGrokInSubquery
344366
required_capability: fork_v9
345367
required_capability: subquery_in_from_command

0 commit comments

Comments
 (0)