Skip to content

Commit f1ff526

Browse files
julian-elasticjulian-elastic
authored
ESQL: Add telemetry support for Lookup Join On Expression (elastic#134942)
Add telemetry support for Lookup Join On Expression, adding LOOKUP JOIN ON EXPRESSION label Keep the existing LOOKUP JOIN telemetry label for lookup join on fields to provide BWC
1 parent 4aa0f16 commit f1ff526

File tree

5 files changed

+259
-169
lines changed

5 files changed

+259
-169
lines changed

‎docs/changelog/134942.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 134942
2+
summary: Add telemetry support for Lookup Join On Expression
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TelemetryIT.java‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -146,6 +146,23 @@ public static Iterable<Object[]> parameters() {
146146
Map.ofEntries(Map.entry("TO_STRING", 1)),
147147
true
148148
) },
149+
new Object[] {
150+
new Test(
151+
"""
152+
FROM idx
153+
| EVAL y = to_str(host)
154+
| RENAME host as host_left
155+
| LOOKUP JOIN lookup_idx ON host_left == host
156+
""",
157+
Map.ofEntries(
158+
Map.entry("RENAME", 1),
159+
Map.entry("FROM", 1),
160+
Map.entry("EVAL", 1),
161+
Map.entry("LOOKUP JOIN ON EXPRESSION", 1)
162+
),
163+
Map.ofEntries(Map.entry("TO_STRING", 1)),
164+
true
165+
) },
149166
new Object[] {
150167
new Test(
151168
"TS time_series_idx | LIMIT 10",
@@ -184,6 +201,12 @@ public void init() {
184201
}
185202

186203
public void testMetrics() throws Exception {
204+
if (testCase.query().contains("LOOKUP JOIN lookup_idx ON host_left == host")) {
205+
assumeTrue(
206+
"requires LOOKUP JOIN ON boolean expression capability",
207+
EsqlCapabilities.Cap.LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION.isEnabled()
208+
);
209+
}
187210
DiscoveryNode dataNode = randomDataNode();
188211
testQuery(dataNode, testCase);
189212
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/telemetry/FeatureMetric.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,8 @@ public enum FeatureMetric {
6363
DROP(Drop.class::isInstance),
6464
KEEP(Keep.class::isInstance),
6565
RENAME(Rename.class::isInstance),
66-
LOOKUP_JOIN(LookupJoin.class::isInstance),
66+
LOOKUP_JOIN(plan -> plan instanceof LookupJoin lookupJoin && lookupJoin.config().joinOnConditions() == null),
67+
LOOKUP_JOIN_ON_EXPRESSION(plan -> plan instanceof LookupJoin lookupJoin && lookupJoin.config().joinOnConditions() != null),
6768
LOOKUP(Lookup.class::isInstance),
6869
CHANGE_POINT(ChangePoint.class::isInstance),
6970
INLINE_STATS(InlineStats.class::isInstance),

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/telemetry/VerifierMetricsTests.java‎

Lines changed: 78 additions & 21 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.KEEP;
3535
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LIMIT;
3636
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LOOKUP_JOIN;
37+
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.LOOKUP_JOIN_ON_EXPRESSION;
3738
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.MV_EXPAND;
3839
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.RENAME;
3940
import static org.elasticsearch.xpack.esql.telemetry.FeatureMetric.ROW;
@@ -65,7 +66,8 @@ public void testDissectQuery() {
6566
assertEquals(0, drop(c));
6667
assertEquals(0, keep(c));
6768
assertEquals(0, rename(c));
68-
assertEquals(0, lookupjoin(c));
69+
assertEquals(0, lookupJoinOnFields(c));
70+
assertEquals(0, lookupJoinOnExpression(c));
6971
assertEquals(0, inlineStats(c));
7072
assertEquals(1, function("concat", c));
7173
}
@@ -87,7 +89,8 @@ public void testEvalQuery() {
8789
assertEquals(0, drop(c));
8890
assertEquals(0, keep(c));
8991
assertEquals(0, rename(c));
90-
assertEquals(0, lookupjoin(c));
92+
assertEquals(0, lookupJoinOnFields(c));
93+
assertEquals(0, lookupJoinOnExpression(c));
9194
assertEquals(0, inlineStats(c));
9295
assertEquals(1, function("length", c));
9396
}
@@ -109,7 +112,8 @@ public void testGrokQuery() {
109112
assertEquals(0, drop(c));
110113
assertEquals(0, keep(c));
111114
assertEquals(0, rename(c));
112-
assertEquals(0, lookupjoin(c));
115+
assertEquals(0, lookupJoinOnFields(c));
116+
assertEquals(0, lookupJoinOnExpression(c));
113117
assertEquals(0, inlineStats(c));
114118
assertEquals(1, function("concat", c));
115119
}
@@ -131,7 +135,8 @@ public void testLimitQuery() {
131135
assertEquals(0, drop(c));
132136
assertEquals(0, keep(c));
133137
assertEquals(0, rename(c));
134-
assertEquals(0, lookupjoin(c));
138+
assertEquals(0, lookupJoinOnFields(c));
139+
assertEquals(0, lookupJoinOnExpression(c));
135140
assertEquals(0, inlineStats(c));
136141
}
137142

@@ -152,7 +157,8 @@ public void testSortQuery() {
152157
assertEquals(0, drop(c));
153158
assertEquals(0, keep(c));
154159
assertEquals(0, rename(c));
155-
assertEquals(0, lookupjoin(c));
160+
assertEquals(0, lookupJoinOnFields(c));
161+
assertEquals(0, lookupJoinOnExpression(c));
156162
assertEquals(0, inlineStats(c));
157163
}
158164

@@ -173,7 +179,8 @@ public void testStatsQuery() {
173179
assertEquals(0, drop(c));
174180
assertEquals(0, keep(c));
175181
assertEquals(0, rename(c));
176-
assertEquals(0, lookupjoin(c));
182+
assertEquals(0, lookupJoinOnFields(c));
183+
assertEquals(0, lookupJoinOnExpression(c));
177184
assertEquals(0, inlineStats(c));
178185
assertEquals(1, function("max", c));
179186
}
@@ -195,7 +202,8 @@ public void testWhereQuery() {
195202
assertEquals(0, drop(c));
196203
assertEquals(0, keep(c));
197204
assertEquals(0, rename(c));
198-
assertEquals(0, lookupjoin(c));
205+
assertEquals(0, lookupJoinOnFields(c));
206+
assertEquals(0, lookupJoinOnExpression(c));
199207
assertEquals(0, inlineStats(c));
200208
}
201209

@@ -216,7 +224,8 @@ public void testTwoWhereQuery() {
216224
assertEquals(0, drop(c));
217225
assertEquals(0, keep(c));
218226
assertEquals(0, rename(c));
219-
assertEquals(0, lookupjoin(c));
227+
assertEquals(0, lookupJoinOnFields(c));
228+
assertEquals(0, lookupJoinOnExpression(c));
220229
assertEquals(0, inlineStats(c));
221230
}
222231

@@ -257,7 +266,8 @@ public void testTwoQueriesExecuted() {
257266
assertEquals(0, drop(c));
258267
assertEquals(0, keep(c));
259268
assertEquals(0, rename(c));
260-
assertEquals(0, lookupjoin(c));
269+
assertEquals(0, lookupJoinOnFields(c));
270+
assertEquals(0, lookupJoinOnExpression(c));
261271
assertEquals(0, inlineStats(c));
262272

263273
assertEquals(1, function("length", c));
@@ -343,7 +353,8 @@ public void testEnrich() {
343353
assertEquals(1L, keep(c));
344354
assertEquals(0, rename(c));
345355
assertEquals(0, inlineStats(c));
346-
assertEquals(0, lookupjoin(c));
356+
assertEquals(0, lookupJoinOnFields(c));
357+
assertEquals(0, lookupJoinOnExpression(c));
347358
assertEquals(1, function("to_string", c));
348359
}
349360

@@ -374,7 +385,8 @@ public void testMvExpand() {
374385
assertEquals(1L, keep(c));
375386
assertEquals(0, rename(c));
376387
assertEquals(0, inlineStats(c));
377-
assertEquals(0, lookupjoin(c));
388+
assertEquals(0, lookupJoinOnFields(c));
389+
assertEquals(0, lookupJoinOnExpression(c));
378390
}
379391

380392
public void testShowInfo() {
@@ -395,7 +407,8 @@ public void testShowInfo() {
395407
assertEquals(0, keep(c));
396408
assertEquals(0, rename(c));
397409
assertEquals(0, inlineStats(c));
398-
assertEquals(0, lookupjoin(c));
410+
assertEquals(0, lookupJoinOnFields(c));
411+
assertEquals(0, lookupJoinOnExpression(c));
399412
assertEquals(1, function("count", c));
400413
}
401414

@@ -417,7 +430,8 @@ public void testRow() {
417430
assertEquals(0, keep(c));
418431
assertEquals(0, rename(c));
419432
assertEquals(0, inlineStats(c));
420-
assertEquals(0, lookupjoin(c));
433+
assertEquals(0, lookupJoinOnFields(c));
434+
assertEquals(0, lookupJoinOnExpression(c));
421435
}
422436

423437
public void testDropAndRename() {
@@ -438,7 +452,8 @@ public void testDropAndRename() {
438452
assertEquals(0, keep(c));
439453
assertEquals(1L, rename(c));
440454
assertEquals(0, inlineStats(c));
441-
assertEquals(0, lookupjoin(c));
455+
assertEquals(0, lookupJoinOnFields(c));
456+
assertEquals(0, lookupJoinOnExpression(c));
442457
assertEquals(1, function("count", c));
443458
}
444459

@@ -465,7 +480,8 @@ public void testKeep() {
465480
assertEquals(1L, keep(c));
466481
assertEquals(0, rename(c));
467482
assertEquals(0, inlineStats(c));
468-
assertEquals(0, lookupjoin(c));
483+
assertEquals(0, lookupJoinOnFields(c));
484+
assertEquals(0, lookupJoinOnExpression(c));
469485
}
470486

471487
public void testCategorize() {
@@ -490,7 +506,8 @@ public void testCategorize() {
490506
assertEquals(1L, keep(c));
491507
assertEquals(0, rename(c));
492508
assertEquals(0, inlineStats(c));
493-
assertEquals(0, lookupjoin(c));
509+
assertEquals(0, lookupJoinOnFields(c));
510+
assertEquals(0, lookupJoinOnExpression(c));
494511
assertEquals(1, function("count", c));
495512
assertEquals(1, function("categorize", c));
496513
}
@@ -517,7 +534,8 @@ public void testInlineStatsStandalone() {
517534
assertEquals(0, keep(c));
518535
assertEquals(0, rename(c));
519536
assertEquals(1L, inlineStats(c));
520-
assertEquals(0, lookupjoin(c));
537+
assertEquals(0, lookupJoinOnFields(c));
538+
assertEquals(0, lookupJoinOnExpression(c));
521539
assertEquals(1, function("max", c));
522540
}
523541

@@ -544,7 +562,8 @@ public void testInlineStatsWithOtherStats() {
544562
assertEquals(0, keep(c));
545563
assertEquals(0, rename(c));
546564
assertEquals(1L, inlineStats(c));
547-
assertEquals(0, lookupjoin(c));
565+
assertEquals(0, lookupJoinOnFields(c));
566+
assertEquals(0, lookupJoinOnExpression(c));
548567
assertEquals(1, function("max", c));
549568
}
550569

@@ -570,7 +589,40 @@ public void testBinaryPlanAfterStats() {
570589
assertEquals(0, keep(c));
571590
assertEquals(0, rename(c));
572591
assertEquals(0, inlineStats(c));
573-
assertEquals(1L, lookupjoin(c));
592+
assertEquals(1L, lookupJoinOnFields(c));
593+
assertEquals(0, lookupJoinOnExpression(c));
594+
assertEquals(1, function("max", c));
595+
}
596+
597+
public void testBinaryPlanAfterStatsExpressionJoin() {
598+
assumeTrue(
599+
"requires LOOKUP JOIN ON boolean expression capability",
600+
EsqlCapabilities.Cap.LOOKUP_JOIN_ON_BOOLEAN_EXPRESSION.isEnabled()
601+
);
602+
Counters c = esql("""
603+
from employees
604+
| eval language_code = languages
605+
| stats m = max(salary) by language_code
606+
| rename language_code as language_code_left
607+
| lookup join languages_lookup on language_code_left >= language_code""");
608+
assertEquals(0, dissect(c));
609+
assertEquals(1L, eval(c));
610+
assertEquals(0, grok(c));
611+
assertEquals(0, limit(c));
612+
assertEquals(0, sort(c));
613+
assertEquals(1L, stats(c));
614+
assertEquals(0, where(c));
615+
assertEquals(0, enrich(c));
616+
assertEquals(0, mvExpand(c));
617+
assertEquals(0, show(c));
618+
assertEquals(0, row(c));
619+
assertEquals(1L, from(c));
620+
assertEquals(0, drop(c));
621+
assertEquals(0, keep(c));
622+
assertEquals(1L, rename(c));
623+
assertEquals(0, inlineStats(c));
624+
assertEquals(0, lookupJoinOnFields(c));
625+
assertEquals(1L, lookupJoinOnExpression(c));
574626
assertEquals(1, function("max", c));
575627
}
576628

@@ -597,7 +649,8 @@ public void testBinaryPlanAfterInlineStats() {
597649
assertEquals(0, keep(c));
598650
assertEquals(0, rename(c));
599651
assertEquals(1L, inlineStats(c));
600-
assertEquals(1L, lookupjoin(c));
652+
assertEquals(1L, lookupJoinOnFields(c));
653+
assertEquals(0, lookupJoinOnExpression(c));
601654
assertEquals(1, function("max", c));
602655
}
603656

@@ -665,10 +718,14 @@ private long inlineStats(Counters c) {
665718
return c.get(FEATURES_PREFIX + INLINE_STATS);
666719
}
667720

668-
private long lookupjoin(Counters c) {
721+
private long lookupJoinOnFields(Counters c) {
669722
return c.get(FEATURES_PREFIX + LOOKUP_JOIN);
670723
}
671724

725+
private long lookupJoinOnExpression(Counters c) {
726+
return c.get(FEATURES_PREFIX + LOOKUP_JOIN_ON_EXPRESSION);
727+
}
728+
672729
private long function(String function, Counters c) {
673730
return c.get(FUNC_PREFIX + function);
674731
}

0 commit comments

Comments
 (0)