Skip to content

Commit 1a9ff77

Browse files
astefanastefan
authored
ESQL: telemetry with inlinestats (#134309)
1 parent a82b5cf commit 1a9ff77

File tree

4 files changed

+178
-9
lines changed

4 files changed

+178
-9
lines changed

‎docs/changelog/134309.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 134309
2+
summary: Telemetry with inlinestats
3+
area: ES|QL
4+
type: bug
5+
issues: []

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Analyzer.java‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -100,6 +100,7 @@
100100
import org.elasticsearch.xpack.esql.plan.logical.EsRelation;
101101
import org.elasticsearch.xpack.esql.plan.logical.Eval;
102102
import org.elasticsearch.xpack.esql.plan.logical.Fork;
103+
import org.elasticsearch.xpack.esql.plan.logical.InlineStats;
103104
import org.elasticsearch.xpack.esql.plan.logical.Insist;
104105
import org.elasticsearch.xpack.esql.plan.logical.Keep;
105106
import org.elasticsearch.xpack.esql.plan.logical.Limit;
@@ -171,6 +172,7 @@
171172
import static org.elasticsearch.xpack.esql.core.type.DataType.VERSION;
172173
import static org.elasticsearch.xpack.esql.core.type.DataType.isTemporalAmount;
173174
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT;
175+
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.STATS;
174176
import static org.elasticsearch.xpack.esql.type.EsqlDataTypeConverter.maybeParseTemporalAmount;
175177

176178
/**
@@ -1401,6 +1403,24 @@ private BitSet gatherPreAnalysisMetrics(LogicalPlan plan, BitSet b) {
14011403
if (plan.collectFirstChildren(Limit.class::isInstance).isEmpty() == false) {
14021404
b.set(LIMIT.ordinal());
14031405
}
1406+
1407+
// count only the Aggregate (STATS command) that is "standalone" not also the one that is part of an INLINESTATS command
1408+
if (plan instanceof Aggregate) {
1409+
b.set(STATS.ordinal());
1410+
} else {
1411+
plan.forEachDownMayReturnEarly((p, breakEarly) -> {
1412+
if (p instanceof InlineStats) {
1413+
return;
1414+
}
1415+
for (var c : p.children()) {
1416+
if (c instanceof Aggregate) {
1417+
b.set(STATS.ordinal());
1418+
breakEarly.set(true);
1419+
return;
1420+
}
1421+
}
1422+
});
1423+
}
14041424
plan.forEachDown(p -> FeatureMetric.set(p, b));
14051425
return b;
14061426
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -51,7 +51,8 @@ public enum FeatureMetric {
5151
GROK(Grok.class::isInstance),
5252
LIMIT(plan -> false), // the limit is checked in Analyzer.gatherPreAnalysisMetrics, because it has a more complex and general check
5353
SORT(OrderBy.class::isInstance),
54-
STATS(Aggregate.class::isInstance),
54+
// the STATS is checked in Analyzer.gatherPreAnalysisMetrics, because it can also be part of an inlinestats command
55+
STATS(plan -> false),
5556
WHERE(Filter.class::isInstance),
5657
ENRICH(Enrich.class::isInstance),
5758
EXPLAIN(Explain.class::isInstance),
@@ -81,7 +82,8 @@ public enum FeatureMetric {
8182
EsqlProject.class,
8283
Project.class,
8384
Limit.class, // LIMIT is managed in another way, see above
84-
FuseScoreEval.class
85+
FuseScoreEval.class,
86+
Aggregate.class // STATS is managed in another way, see above
8587
);
8688

8789
private Predicate<LogicalPlan> planCheck;

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java‎

Lines changed: 149 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@
1111
import org.elasticsearch.test.ESTestCase;
1212
import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
1313
import org.elasticsearch.xpack.esql.EsqlTestUtils;
14+
import org.elasticsearch.xpack.esql.action.EsqlCapabilities;
1415
import org.elasticsearch.xpack.esql.analysis.Verifier;
1516
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
1617
import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition;
@@ -29,8 +30,10 @@
2930
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.EVAL;
3031
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.FROM;
3132
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.GROK;
33+
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.INLINESTATS;
3234
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.KEEP;
3335
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT;
36+
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LOOKUP_JOIN;
3437
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.MV_EXPAND;
3538
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.RENAME;
3639
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.ROW;
@@ -62,7 +65,8 @@ public void testDissectQuery() {
6265
assertEquals(0, drop(c));
6366
assertEquals(0, keep(c));
6467
assertEquals(0, rename(c));
65-
68+
assertEquals(0, lookupjoin(c));
69+
assertEquals(0, inlinestats(c));
6670
assertEquals(1, function("concat", c));
6771
}
6872

@@ -83,7 +87,8 @@ public void testEvalQuery() {
8387
assertEquals(0, drop(c));
8488
assertEquals(0, keep(c));
8589
assertEquals(0, rename(c));
86-
90+
assertEquals(0, lookupjoin(c));
91+
assertEquals(0, inlinestats(c));
8792
assertEquals(1, function("length", c));
8893
}
8994

@@ -104,7 +109,8 @@ public void testGrokQuery() {
104109
assertEquals(0, drop(c));
105110
assertEquals(0, keep(c));
106111
assertEquals(0, rename(c));
107-
112+
assertEquals(0, lookupjoin(c));
113+
assertEquals(0, inlinestats(c));
108114
assertEquals(1, function("concat", c));
109115
}
110116

@@ -125,6 +131,8 @@ public void testLimitQuery() {
125131
assertEquals(0, drop(c));
126132
assertEquals(0, keep(c));
127133
assertEquals(0, rename(c));
134+
assertEquals(0, lookupjoin(c));
135+
assertEquals(0, inlinestats(c));
128136
}
129137

130138
public void testSortQuery() {
@@ -144,6 +152,8 @@ public void testSortQuery() {
144152
assertEquals(0, drop(c));
145153
assertEquals(0, keep(c));
146154
assertEquals(0, rename(c));
155+
assertEquals(0, lookupjoin(c));
156+
assertEquals(0, inlinestats(c));
147157
}
148158

149159
public void testStatsQuery() {
@@ -163,7 +173,8 @@ public void testStatsQuery() {
163173
assertEquals(0, drop(c));
164174
assertEquals(0, keep(c));
165175
assertEquals(0, rename(c));
166-
176+
assertEquals(0, lookupjoin(c));
177+
assertEquals(0, inlinestats(c));
167178
assertEquals(1, function("max", c));
168179
}
169180

@@ -184,6 +195,8 @@ public void testWhereQuery() {
184195
assertEquals(0, drop(c));
185196
assertEquals(0, keep(c));
186197
assertEquals(0, rename(c));
198+
assertEquals(0, lookupjoin(c));
199+
assertEquals(0, inlinestats(c));
187200
}
188201

189202
public void testTwoWhereQuery() {
@@ -203,6 +216,8 @@ public void testTwoWhereQuery() {
203216
assertEquals(0, drop(c));
204217
assertEquals(0, keep(c));
205218
assertEquals(0, rename(c));
219+
assertEquals(0, lookupjoin(c));
220+
assertEquals(0, inlinestats(c));
206221
}
207222

208223
public void testTwoQueriesExecuted() {
@@ -242,6 +257,8 @@ public void testTwoQueriesExecuted() {
242257
assertEquals(0, drop(c));
243258
assertEquals(0, keep(c));
244259
assertEquals(0, rename(c));
260+
assertEquals(0, lookupjoin(c));
261+
assertEquals(0, inlinestats(c));
245262

246263
assertEquals(1, function("length", c));
247264
assertEquals(1, function("concat", c));
@@ -325,7 +342,8 @@ public void testEnrich() {
325342
assertEquals(0, drop(c));
326343
assertEquals(1L, keep(c));
327344
assertEquals(0, rename(c));
328-
345+
assertEquals(0, inlinestats(c));
346+
assertEquals(0, lookupjoin(c));
329347
assertEquals(1, function("to_string", c));
330348
}
331349

@@ -355,6 +373,8 @@ public void testMvExpand() {
355373
assertEquals(0, drop(c));
356374
assertEquals(1L, keep(c));
357375
assertEquals(0, rename(c));
376+
assertEquals(0, inlinestats(c));
377+
assertEquals(0, lookupjoin(c));
358378
}
359379

360380
public void testShowInfo() {
@@ -374,7 +394,8 @@ public void testShowInfo() {
374394
assertEquals(0, drop(c));
375395
assertEquals(0, keep(c));
376396
assertEquals(0, rename(c));
377-
397+
assertEquals(0, inlinestats(c));
398+
assertEquals(0, lookupjoin(c));
378399
assertEquals(1, function("count", c));
379400
}
380401

@@ -395,6 +416,8 @@ public void testRow() {
395416
assertEquals(0, drop(c));
396417
assertEquals(0, keep(c));
397418
assertEquals(0, rename(c));
419+
assertEquals(0, inlinestats(c));
420+
assertEquals(0, lookupjoin(c));
398421
}
399422

400423
public void testDropAndRename() {
@@ -414,7 +437,8 @@ public void testDropAndRename() {
414437
assertEquals(1L, drop(c));
415438
assertEquals(0, keep(c));
416439
assertEquals(1L, rename(c));
417-
440+
assertEquals(0, inlinestats(c));
441+
assertEquals(0, lookupjoin(c));
418442
assertEquals(1, function("count", c));
419443
}
420444

@@ -440,6 +464,8 @@ public void testKeep() {
440464
assertEquals(0, drop(c));
441465
assertEquals(1L, keep(c));
442466
assertEquals(0, rename(c));
467+
assertEquals(0, inlinestats(c));
468+
assertEquals(0, lookupjoin(c));
443469
}
444470

445471
public void testCategorize() {
@@ -463,10 +489,118 @@ public void testCategorize() {
463489
assertEquals(0, drop(c));
464490
assertEquals(1L, keep(c));
465491
assertEquals(0, rename(c));
492+
assertEquals(0, inlinestats(c));
493+
assertEquals(0, lookupjoin(c));
466494
assertEquals(1, function("count", c));
467495
assertEquals(1, function("categorize", c));
468496
}
469497

498+
public void testInlinestatsStandalone() {
499+
assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled());
500+
Counters c = esql("""
501+
from employees
502+
| inlinestats max(salary) by gender
503+
| where languages is not null""");
504+
assertEquals(0, dissect(c));
505+
assertEquals(0, eval(c));
506+
assertEquals(0, grok(c));
507+
assertEquals(0, limit(c));
508+
assertEquals(0, sort(c));
509+
assertEquals(0, stats(c));
510+
assertEquals(1L, where(c));
511+
assertEquals(0, enrich(c));
512+
assertEquals(0, mvExpand(c));
513+
assertEquals(0, show(c));
514+
assertEquals(0, row(c));
515+
assertEquals(1L, from(c));
516+
assertEquals(0, drop(c));
517+
assertEquals(0, keep(c));
518+
assertEquals(0, rename(c));
519+
assertEquals(1L, inlinestats(c));
520+
assertEquals(0, lookupjoin(c));
521+
assertEquals(1, function("max", c));
522+
}
523+
524+
public void testInlinestatsWithOtherStats() {
525+
assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled());
526+
Counters c = esql("""
527+
from employees
528+
| inlinestats m = max(salary) by gender
529+
| where languages is not null
530+
| stats max(m) by languages""");
531+
assertEquals(0, dissect(c));
532+
assertEquals(0, eval(c));
533+
assertEquals(0, grok(c));
534+
assertEquals(0, limit(c));
535+
assertEquals(0, sort(c));
536+
assertEquals(1L, stats(c));
537+
assertEquals(1L, where(c));
538+
assertEquals(0, enrich(c));
539+
assertEquals(0, mvExpand(c));
540+
assertEquals(0, show(c));
541+
assertEquals(0, row(c));
542+
assertEquals(1L, from(c));
543+
assertEquals(0, drop(c));
544+
assertEquals(0, keep(c));
545+
assertEquals(0, rename(c));
546+
assertEquals(1L, inlinestats(c));
547+
assertEquals(0, lookupjoin(c));
548+
assertEquals(1, function("max", c));
549+
}
550+
551+
public void testBinaryPlanAfterStats() {
552+
Counters c = esql("""
553+
from employees
554+
| eval language_code = languages
555+
| stats m = max(salary) by language_code
556+
| lookup join languages_lookup on language_code""");
557+
assertEquals(0, dissect(c));
558+
assertEquals(1L, eval(c));
559+
assertEquals(0, grok(c));
560+
assertEquals(0, limit(c));
561+
assertEquals(0, sort(c));
562+
assertEquals(1L, stats(c));
563+
assertEquals(0, where(c));
564+
assertEquals(0, enrich(c));
565+
assertEquals(0, mvExpand(c));
566+
assertEquals(0, show(c));
567+
assertEquals(0, row(c));
568+
assertEquals(1L, from(c));
569+
assertEquals(0, drop(c));
570+
assertEquals(0, keep(c));
571+
assertEquals(0, rename(c));
572+
assertEquals(0, inlinestats(c));
573+
assertEquals(1L, lookupjoin(c));
574+
assertEquals(1, function("max", c));
575+
}
576+
577+
public void testBinaryPlanAfterInlinestats() {
578+
assumeTrue("INLINESTATS required", EsqlCapabilities.Cap.INLINESTATS_V11.isEnabled());
579+
Counters c = esql("""
580+
from employees
581+
| eval language_code = languages
582+
| inlinestats m = max(salary) by language_code
583+
| lookup join languages_lookup on language_code""");
584+
assertEquals(0, dissect(c));
585+
assertEquals(1L, eval(c));
586+
assertEquals(0, grok(c));
587+
assertEquals(0, limit(c));
588+
assertEquals(0, sort(c));
589+
assertEquals(0, stats(c));
590+
assertEquals(0, where(c));
591+
assertEquals(0, enrich(c));
592+
assertEquals(0, mvExpand(c));
593+
assertEquals(0, show(c));
594+
assertEquals(0, row(c));
595+
assertEquals(1L, from(c));
596+
assertEquals(0, drop(c));
597+
assertEquals(0, keep(c));
598+
assertEquals(0, rename(c));
599+
assertEquals(1L, inlinestats(c));
600+
assertEquals(1L, lookupjoin(c));
601+
assertEquals(1, function("max", c));
602+
}
603+
470604
private long dissect(Counters c) {
471605
return c.get(FEATURES_PREFIX + DISSECT);
472606
}
@@ -527,6 +661,14 @@ private long rename(Counters c) {
527661
return c.get(FEATURES_PREFIX + RENAME);
528662
}
529663

664+
private long inlinestats(Counters c) {
665+
return c.get(FEATURES_PREFIX + INLINESTATS);
666+
}
667+
668+
private long lookupjoin(Counters c) {
669+
return c.get(FEATURES_PREFIX + LOOKUP_JOIN);
670+
}
671+
530672
private long function(String function, Counters c) {
531673
return c.get(FUNC_PREFIX + function);
532674
}

0 commit comments

Comments
 (0)