Skip to content

Commit 9b194d6

Browse files
luigidellaquilaluigidellaquila
committed
ES|QL: add metrics for functions (#114620)
1 parent b80f677 commit 9b194d6

File tree

11 files changed

+171
-11
lines changed

11 files changed

+171
-11
lines changed

‎docs/changelog/114620.yaml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
pr: 114620
2+
summary: "ES|QL: add metrics for functions"
3+
area: ES|QL
4+
type: enhancement
5+
issues: []

‎docs/reference/rest-api/usage.asciidoc‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,9 +38,10 @@ include::{es-ref-dir}/rest-api/common-parms.asciidoc[tag=master-timeout]
3838
------------------------------------------------------------
3939
GET /_xpack/usage
4040
------------------------------------------------------------
41-
// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*/]
41+
// TEST[s/usage/usage?filter_path=-watcher.execution.actions.index*\,-watcher.execution.actions.logging*,-watcher.execution.actions.email*,-esql.functions*/]
4242
// This response filter removes watcher logging results if they are included
4343
// to avoid errors in the CI builds.
44+
// Same for ES|QL functions, that is a long list and quickly evolving.
4445

4546
[source,console-result]
4647
------------------------------------------------------------

‎x-pack/plugin/esql/qa/testFixtures/src/main/java/org/elasticsearch/xpack/esql/EsqlTestUtils.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.elasticsearch.xpack.esql.core.type.EsField;
4747
import org.elasticsearch.xpack.esql.core.util.DateUtils;
4848
import org.elasticsearch.xpack.esql.core.util.StringUtils;
49+
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
4950
import org.elasticsearch.xpack.esql.expression.function.scalar.string.RLike;
5051
import org.elasticsearch.xpack.esql.expression.function.scalar.string.WildcardLike;
5152
import org.elasticsearch.xpack.esql.expression.predicate.operator.comparison.Equals;
@@ -260,7 +261,7 @@ public boolean isIndexed(String field) {
260261

261262
public static final Configuration TEST_CFG = configuration(new QueryPragmas(Settings.EMPTY));
262263

263-
public static final Verifier TEST_VERIFIER = new Verifier(new Metrics());
264+
public static final Verifier TEST_VERIFIER = new Verifier(new Metrics(new EsqlFunctionRegistry()));
264265

265266
private EsqlTestUtils() {}
266267

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/action/EsqlCapabilities.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -390,7 +390,12 @@ public enum Cap {
390390
/**
391391
* Fix for https://github.com/elastic/elasticsearch/issues/114714
392392
*/
393-
FIX_STATS_BY_FOLDABLE_EXPRESSION;
393+
FIX_STATS_BY_FOLDABLE_EXPRESSION,
394+
395+
/**
396+
* Adding stats for functions (stack telemetry)
397+
*/
398+
FUNCTION_STATS;
394399

395400
private final boolean snapshotOnly;
396401
private final FeatureFlag featureFlag;

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/analysis/Verifier.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@
5858
import java.util.ArrayList;
5959
import java.util.BitSet;
6060
import java.util.Collection;
61+
import java.util.HashSet;
6162
import java.util.LinkedHashSet;
6263
import java.util.List;
6364
import java.util.Locale;
@@ -480,6 +481,9 @@ private void gatherMetrics(LogicalPlan plan, BitSet b) {
480481
for (int i = b.nextSetBit(0); i >= 0; i = b.nextSetBit(i + 1)) {
481482
metrics.inc(FeatureMetric.values()[i]);
482483
}
484+
Set<Class<?>> functions = new HashSet<>();
485+
plan.forEachExpressionDown(Function.class, p -> functions.add(p.getClass()));
486+
functions.forEach(f -> metrics.incFunctionMetric(f));
483487
}
484488

485489
/**

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/execution/PlanExecutor.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -48,7 +48,7 @@ public PlanExecutor(IndexResolver indexResolver, MeterRegistry meterRegistry) {
4848
this.preAnalyzer = new PreAnalyzer();
4949
this.functionRegistry = new EsqlFunctionRegistry();
5050
this.mapper = new Mapper(functionRegistry);
51-
this.metrics = new Metrics();
51+
this.metrics = new Metrics(functionRegistry);
5252
this.verifier = new Verifier(metrics);
5353
this.planningMetricsManager = new PlanningMetricsManager(meterRegistry);
5454
}

‎x-pack/plugin/esql/src/main/java/org/elasticsearch/xpack/esql/stats/Metrics.java‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -10,8 +10,11 @@
1010
import org.elasticsearch.common.metrics.CounterMetric;
1111
import org.elasticsearch.common.util.Maps;
1212
import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
13+
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
14+
import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition;
1315

1416
import java.util.Collections;
17+
import java.util.HashMap;
1518
import java.util.LinkedHashMap;
1619
import java.util.Locale;
1720
import java.util.Map;
@@ -36,10 +39,17 @@ public String toString() {
3639
private final Map<QueryMetric, Map<OperationType, CounterMetric>> opsByTypeMetrics;
3740
// map that holds one counter per esql query "feature" (eval, sort, limit, where....)
3841
private final Map<FeatureMetric, CounterMetric> featuresMetrics;
42+
private final Map<String, CounterMetric> functionMetrics;
3943
protected static String QPREFIX = "queries.";
4044
protected static String FPREFIX = "features.";
45+
protected static String FUNC_PREFIX = "functions.";
4146

42-
public Metrics() {
47+
private final EsqlFunctionRegistry functionRegistry;
48+
private final Map<Class<?>, String> classToFunctionName;
49+
50+
public Metrics(EsqlFunctionRegistry functionRegistry) {
51+
this.functionRegistry = functionRegistry.snapshotRegistry();
52+
this.classToFunctionName = initClassToFunctionType();
4353
Map<QueryMetric, Map<OperationType, CounterMetric>> qMap = new LinkedHashMap<>();
4454
for (QueryMetric metric : QueryMetric.values()) {
4555
Map<OperationType, CounterMetric> metricsMap = Maps.newLinkedHashMapWithExpectedSize(OperationType.values().length);
@@ -56,6 +66,26 @@ public Metrics() {
5666
fMap.put(featureMetric, new CounterMetric());
5767
}
5868
featuresMetrics = Collections.unmodifiableMap(fMap);
69+
70+
functionMetrics = initFunctionMetrics();
71+
}
72+
73+
private Map<String, CounterMetric> initFunctionMetrics() {
74+
Map<String, CounterMetric> result = new LinkedHashMap<>();
75+
for (var entry : classToFunctionName.entrySet()) {
76+
result.put(entry.getValue(), new CounterMetric());
77+
}
78+
return Collections.unmodifiableMap(result);
79+
}
80+
81+
private Map<Class<?>, String> initClassToFunctionType() {
82+
Map<Class<?>, String> tmp = new HashMap<>();
83+
for (FunctionDefinition func : functionRegistry.listFunctions()) {
84+
if (tmp.containsKey(func.clazz()) == false) {
85+
tmp.put(func.clazz(), func.name());
86+
}
87+
}
88+
return Collections.unmodifiableMap(tmp);
5989
}
6090

6191
/**
@@ -81,6 +111,13 @@ public void inc(FeatureMetric metric) {
81111
this.featuresMetrics.get(metric).inc();
82112
}
83113

114+
public void incFunctionMetric(Class<?> functionType) {
115+
String functionName = classToFunctionName.get(functionType);
116+
if (functionName != null) {
117+
functionMetrics.get(functionName).inc();
118+
}
119+
}
120+
84121
public Counters stats() {
85122
Counters counters = new Counters();
86123

@@ -102,6 +139,11 @@ public Counters stats() {
102139
counters.inc(FPREFIX + entry.getKey().toString(), entry.getValue().count());
103140
}
104141

142+
// function metrics
143+
for (Entry<String, CounterMetric> entry : functionMetrics.entrySet()) {
144+
counters.inc(FUNC_PREFIX + entry.getKey(), entry.getValue().count());
145+
}
146+
105147
return counters;
106148
}
107149
}

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/optimizer/LocalPhysicalPlanOptimizerTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -143,7 +143,7 @@ private Analyzer makeAnalyzer(String mappingFileName, EnrichResolution enrichRes
143143

144144
return new Analyzer(
145145
new AnalyzerContext(config, new EsqlFunctionRegistry(), getIndexResult, enrichResolution),
146-
new Verifier(new Metrics())
146+
new Verifier(new Metrics(new EsqlFunctionRegistry()))
147147
);
148148
}
149149

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/planner/QueryTranslatorTests.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -46,7 +46,7 @@ private static Analyzer makeAnalyzer(String mappingFileName) {
4646

4747
return new Analyzer(
4848
new AnalyzerContext(EsqlTestUtils.TEST_CFG, new EsqlFunctionRegistry(), getIndexResult, new EnrichResolution()),
49-
new Verifier(new Metrics())
49+
new Verifier(new Metrics(new EsqlFunctionRegistry()))
5050
);
5151
}
5252

‎x-pack/plugin/esql/src/test/java/org/elasticsearch/xpack/esql/stats/VerifierMetricsTests.java‎

Lines changed: 93 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -10,9 +10,14 @@
1010
import org.elasticsearch.test.ESTestCase;
1111
import org.elasticsearch.xpack.core.watcher.common.stats.Counters;
1212
import org.elasticsearch.xpack.esql.analysis.Verifier;
13+
import org.elasticsearch.xpack.esql.expression.function.EsqlFunctionRegistry;
14+
import org.elasticsearch.xpack.esql.expression.function.FunctionDefinition;
1315
import org.elasticsearch.xpack.esql.parser.EsqlParser;
1416

17+
import java.util.HashMap;
1518
import java.util.List;
19+
import java.util.Map;
20+
import java.util.Set;
1621

1722
import static org.elasticsearch.xpack.esql.EsqlTestUtils.withDefaultLimitWarning;
1823
import static org.elasticsearch.xpack.esql.analysis.AnalyzerTestUtils.analyzer;
@@ -32,6 +37,7 @@
3237
import static org.elasticsearch.xpack.esql.stats.FeatureMetric.STATS;
3338
import static org.elasticsearch.xpack.esql.stats.FeatureMetric.WHERE;
3439
import static org.elasticsearch.xpack.esql.stats.Metrics.FPREFIX;
40+
import static org.elasticsearch.xpack.esql.stats.Metrics.FUNC_PREFIX;
3541

3642
public class VerifierMetricsTests extends ESTestCase {
3743

@@ -54,6 +60,8 @@ public void testDissectQuery() {
5460
assertEquals(0, drop(c));
5561
assertEquals(0, keep(c));
5662
assertEquals(0, rename(c));
63+
64+
assertEquals(1, function("concat", c));
5765
}
5866

5967
public void testEvalQuery() {
@@ -73,6 +81,8 @@ public void testEvalQuery() {
7381
assertEquals(0, drop(c));
7482
assertEquals(0, keep(c));
7583
assertEquals(0, rename(c));
84+
85+
assertEquals(1, function("length", c));
7686
}
7787

7888
public void testGrokQuery() {
@@ -92,6 +102,8 @@ public void testGrokQuery() {
92102
assertEquals(0, drop(c));
93103
assertEquals(0, keep(c));
94104
assertEquals(0, rename(c));
105+
106+
assertEquals(1, function("concat", c));
95107
}
96108

97109
public void testLimitQuery() {
@@ -149,6 +161,8 @@ public void testStatsQuery() {
149161
assertEquals(0, drop(c));
150162
assertEquals(0, keep(c));
151163
assertEquals(0, rename(c));
164+
165+
assertEquals(1, function("max", c));
152166
}
153167

154168
public void testWhereQuery() {
@@ -190,7 +204,7 @@ public void testTwoWhereQuery() {
190204
}
191205

192206
public void testTwoQueriesExecuted() {
193-
Metrics metrics = new Metrics();
207+
Metrics metrics = new Metrics(new EsqlFunctionRegistry());
194208
Verifier verifier = new Verifier(metrics);
195209
esqlWithVerifier("""
196210
from employees
@@ -226,6 +240,64 @@ public void testTwoQueriesExecuted() {
226240
assertEquals(0, drop(c));
227241
assertEquals(0, keep(c));
228242
assertEquals(0, rename(c));
243+
244+
assertEquals(1, function("length", c));
245+
assertEquals(1, function("concat", c));
246+
assertEquals(1, function("max", c));
247+
assertEquals(1, function("min", c));
248+
249+
assertEquals(0, function("sin", c));
250+
assertEquals(0, function("cos", c));
251+
}
252+
253+
public void testMultipleFunctions() {
254+
Metrics metrics = new Metrics(new EsqlFunctionRegistry());
255+
Verifier verifier = new Verifier(metrics);
256+
esqlWithVerifier("""
257+
from employees
258+
| where languages > 2
259+
| limit 5
260+
| eval name_len = length(first_name), surname_len = length(last_name)
261+
| sort length(first_name)
262+
| limit 3
263+
""", verifier);
264+
265+
Counters c = metrics.stats();
266+
assertEquals(1, function("length", c));
267+
assertEquals(0, function("concat", c));
268+
269+
esqlWithVerifier("""
270+
from employees
271+
| where languages > 2
272+
| sort first_name desc nulls first
273+
| dissect concat(first_name, " ", last_name) "%{a} %{b}"
274+
| grok concat(first_name, " ", last_name) "%{WORD:a} %{WORD:b}"
275+
| eval name_len = length(first_name), surname_len = length(last_name)
276+
| stats x = max(languages)
277+
| sort x
278+
| stats y = min(x) by x
279+
""", verifier);
280+
c = metrics.stats();
281+
282+
assertEquals(2, function("length", c));
283+
assertEquals(1, function("concat", c));
284+
assertEquals(1, function("max", c));
285+
assertEquals(1, function("min", c));
286+
287+
EsqlFunctionRegistry fr = new EsqlFunctionRegistry().snapshotRegistry();
288+
Map<Class<?>, String> functions = new HashMap<>();
289+
for (FunctionDefinition func : fr.listFunctions()) {
290+
if (functions.containsKey(func.clazz()) == false) {
291+
functions.put(func.clazz(), func.name());
292+
}
293+
}
294+
for (String value : functions.values()) {
295+
if (Set.of("length", "concat", "max", "min").contains(value) == false) {
296+
assertEquals(0, function(value, c));
297+
}
298+
}
299+
Map<?, ?> map = (Map<?, ?>) c.toNestedMap().get("functions");
300+
assertEquals(functions.size(), map.size());
229301
}
230302

231303
public void testEnrich() {
@@ -251,6 +323,8 @@ public void testEnrich() {
251323
assertEquals(0, drop(c));
252324
assertEquals(1L, keep(c));
253325
assertEquals(0, rename(c));
326+
327+
assertEquals(1, function("to_string", c));
254328
}
255329

256330
public void testMvExpand() {
@@ -298,6 +372,8 @@ public void testShowInfo() {
298372
assertEquals(0, drop(c));
299373
assertEquals(0, keep(c));
300374
assertEquals(0, rename(c));
375+
376+
assertEquals(1, function("count", c));
301377
}
302378

303379
public void testRow() {
@@ -336,6 +412,8 @@ public void testDropAndRename() {
336412
assertEquals(1L, drop(c));
337413
assertEquals(0, keep(c));
338414
assertEquals(1L, rename(c));
415+
416+
assertEquals(1, function("count", c));
339417
}
340418

341419
public void testKeep() {
@@ -422,6 +500,19 @@ private long rename(Counters c) {
422500
return c.get(FPREFIX + RENAME);
423501
}
424502

503+
private long function(String function, Counters c) {
504+
return c.get(FUNC_PREFIX + function);
505+
}
506+
507+
private void assertNullFunction(String function, Counters c) {
508+
try {
509+
c.get(FUNC_PREFIX + function);
510+
fail();
511+
} catch (NullPointerException npe) {
512+
513+
}
514+
}
515+
425516
private Counters esql(String esql) {
426517
return esql(esql, null);
427518
}
@@ -434,7 +525,7 @@ private Counters esql(String esql, Verifier v) {
434525
Verifier verifier = v;
435526
Metrics metrics = null;
436527
if (v == null) {
437-
metrics = new Metrics();
528+
metrics = new Metrics(new EsqlFunctionRegistry());
438529
verifier = new Verifier(metrics);
439530
}
440531
analyzer(verifier).analyze(parser.createStatement(esql));

0 commit comments

Comments
 (0)