Skip to content

Commit c8de8d1

Browse files
dnhatndnhatn
authored
Update metrics command (elastic#125193)
This change updates the METRICS command syntax from METRICS index_pattern aggregations | ... to METRICS index_pattern | ..., removing inline aggregations. This PR does not filter out non-time-series indexes from the source; this will be addressed in a follow-up PR to keep this PR small.
1 parent 44a1190 commit c8de8d1

31 files changed

+2636
-2835
lines changed

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-metrics.csv-spec‎

Lines changed: 25 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,5 @@
11
metricsWithoutAggs
2-
required_capability: metrics_syntax
2+
required_capability: metrics_command
33
METRICS k8s | sort @timestamp DESC, cluster, pod | keep @timestamp,cluster,pod,network.bytes_in,network.cost | limit 5;
44

55
@timestamp:datetime | cluster:keyword | pod: keyword| network.bytes_in:long | network.cost:double
@@ -11,9 +11,9 @@ METRICS k8s | sort @timestamp DESC, cluster, pod | keep @timestamp,cluster,pod,n
1111
;
1212

1313
metricsWithAggsAndSourceQuoting
14-
required_capability: metrics_syntax
14+
required_capability: metrics_command
1515
required_capability: double_quotes_source_enclosing
16-
METRICS "k8s" max_bytes=max(to_long(network.total_bytes_in)) BY cluster | SORT max_bytes DESC;
16+
METRICS "k8s" | STATS max_bytes=max(to_long(network.total_bytes_in)) BY cluster | SORT max_bytes DESC;
1717

1818
max_bytes:long | cluster: keyword
1919
10797 | qa
@@ -22,49 +22,49 @@ max_bytes:long | cluster: keyword
2222
;
2323

2424
maxRateAndSourceTripleQuoting
25-
required_capability: metrics_syntax
25+
required_capability: metrics_command
2626
required_capability: double_quotes_source_enclosing
27-
METRICS """k8s""" max(rate(network.total_bytes_in, 1minute));
27+
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute));
2828

2929
max(rate(network.total_bytes_in, 1minute)): double
3030
790.4235090751945
3131
;
3232

3333
maxCost
34-
required_capability: metrics_syntax
35-
METRICS k8s max_cost=max(rate(network.total_cost));
34+
required_capability: metrics_command
35+
METRICS k8s | STATS max_cost=max(rate(network.total_cost));
3636

3737
max_cost: double
3838
0.16151685393258428
3939
;
4040

4141
maxRateAndBytes
42-
required_capability: metrics_syntax
43-
METRICS k8s max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in);
42+
required_capability: metrics_command
43+
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in);
4444

4545
max(rate(network.total_bytes_in, 1minute)): double | max(network.bytes_in): long
4646
790.4235090751945 | 1021
4747
;
4848

4949
`maxRateAndMarkupBytes`
50-
required_capability: metrics_syntax
51-
METRICS k8s max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in * 1.05);
50+
required_capability: metrics_command
51+
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in * 1.05);
5252

5353
max(rate(network.total_bytes_in, 1minute)): double | max(network.bytes_in * 1.05): double
5454
790.4235090751945 | 1072.05
5555
;
5656

5757
maxRateAndBytesAndCost
58-
required_capability: metrics_syntax
59-
METRICS k8s max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in), max(rate(network.total_cost));
58+
required_capability: metrics_command
59+
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in), max(rate(network.total_cost));
6060

6161
max(rate(network.total_bytes_in, 1minute)): double| max(network.bytes_in): long| max(rate(network.total_cost)): double
6262
790.4235090751945 | 1021 | 0.16151685393258428
6363
;
6464

6565
sumRate
66-
required_capability: metrics_syntax
67-
METRICS k8s bytes=sum(rate(network.total_bytes_in)), sum(rate(network.total_cost)) BY cluster | SORT cluster;
66+
required_capability: metrics_command
67+
METRICS k8s | STATS bytes=sum(rate(network.total_bytes_in)), sum(rate(network.total_cost)) BY cluster | SORT cluster;
6868

6969
bytes: double | sum(rate(network.total_cost)): double | cluster: keyword
7070
24.49149357711476 | 0.3018995503437827 | prod
@@ -73,17 +73,17 @@ bytes: double | sum(rate(network.total_cost)): double | cluster: keyword
7373
;
7474

7575
oneRateWithBucket
76-
required_capability: metrics_syntax
77-
METRICS k8s max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 2;
76+
required_capability: metrics_command
77+
METRICS k8s | STATS max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 2;
7878

7979
max(rate(network.total_bytes_in)): double | time_bucket:date
8080
10.594594594594595 | 2024-05-10T00:20:00.000Z
8181
23.702205882352942 | 2024-05-10T00:15:00.000Z
8282
;
8383

8484
twoRatesWithBucket
85-
required_capability: metrics_syntax
86-
METRICS k8s max(rate(network.total_bytes_in)), sum(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 3;
85+
required_capability: metrics_command
86+
METRICS k8s | STATS max(rate(network.total_bytes_in)), sum(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 3;
8787

8888
max(rate(network.total_bytes_in)): double | sum(rate(network.total_bytes_in)): double | time_bucket:date
8989
10.594594594594595 | 42.70864495221802 | 2024-05-10T00:20:00.000Z
@@ -93,8 +93,8 @@ max(rate(network.total_bytes_in)): double | sum(rate(network.total_bytes_in)): d
9393

9494

9595
oneRateWithBucketAndCluster
96-
required_capability: metrics_syntax
97-
METRICS k8s max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC, cluster | LIMIT 6;
96+
required_capability: metrics_command
97+
METRICS k8s | STATS max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC, cluster | LIMIT 6;
9898

9999
max(rate(network.total_bytes_in)): double | time_bucket:date | cluster: keyword
100100
10.594594594594595 | 2024-05-10T00:20:00.000Z | prod
@@ -106,8 +106,8 @@ max(rate(network.total_bytes_in)): double | time_bucket:date | cluster:
106106
;
107107

108108
BytesAndCostByBucketAndCluster
109-
required_capability: metrics_syntax
110-
METRICS k8s max(rate(network.total_bytes_in)), max(network.cost) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC, cluster | LIMIT 6;
109+
required_capability: metrics_command
110+
METRICS k8s | STATS max(rate(network.total_bytes_in)), max(network.cost) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC, cluster | LIMIT 6;
111111

112112
max(rate(network.total_bytes_in)): double | max(network.cost): double | time_bucket:date | cluster: keyword
113113
10.594594594594595 | 10.75 | 2024-05-10T00:20:00.000Z | prod
@@ -119,8 +119,8 @@ max(rate(network.total_bytes_in)): double | max(network.cost): double | time_buc
119119
;
120120

121121
oneRateWithBucketAndClusterThenFilter
122-
required_capability: metrics_syntax
123-
METRICS k8s max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | WHERE cluster=="prod" | SORT time_bucket DESC | LIMIT 3;
122+
required_capability: metrics_command
123+
METRICS k8s | WHERE cluster=="prod" | STATS max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC | LIMIT 3;
124124

125125
max(rate(network.total_bytes_in)): double | time_bucket:date | cluster: keyword
126126
10.594594594594595 | 2024-05-10T00:20:00.000Z | prod

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TelemetryIT.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -155,8 +155,10 @@ public static Iterable<Object[]> parameters() {
155155
) },
156156
new Object[] {
157157
new Test(
158-
"METRICS idx max(id) BY host | LIMIT 10",
159-
Build.current().isSnapshot() ? Map.ofEntries(Map.entry("METRICS", 1), Map.entry("LIMIT", 1)) : Collections.emptyMap(),
158+
"METRICS idx | STATS max(id) BY host | LIMIT 10",
159+
Build.current().isSnapshot()
160+
? Map.ofEntries(Map.entry("METRICS", 1), Map.entry("STATS", 1), Map.entry("LIMIT", 1))
161+
: Collections.emptyMap(),
160162
Build.current().isSnapshot() ? Map.ofEntries(Map.entry("MAX", 1)) : Collections.emptyMap(),
161163
Build.current().isSnapshot()
162164
) }

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java‎

Lines changed: 31 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -148,7 +148,7 @@ public void populateIndex() {
148148
public void testSimpleMetrics() {
149149
List<String> sortedGroups = docs.stream().map(d -> d.host).distinct().sorted().toList();
150150
client().admin().indices().prepareRefresh("hosts").get();
151-
try (EsqlQueryResponse resp = run("METRICS hosts load=avg(cpu) BY host | SORT host")) {
151+
try (EsqlQueryResponse resp = run("METRICS hosts | STATS load=avg(cpu) BY host | SORT host")) {
152152
List<List<Object>> rows = EsqlTestUtils.getValuesList(resp);
153153
assertThat(rows, hasSize(sortedGroups.size()));
154154
for (int i = 0; i < rows.size(); i++) {
@@ -195,14 +195,14 @@ record RateKey(String cluster, String host) {
195195
rates.add(v);
196196
}
197197
}
198-
try (var resp = run("METRICS hosts sum(rate(request_count, 1second))")) {
198+
try (var resp = run("METRICS hosts | STATS sum(rate(request_count, 1second))")) {
199199
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count, 1second))", "double"))));
200200
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
201201
assertThat(values, hasSize(1));
202202
assertThat(values.get(0), hasSize(1));
203203
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).sum(), 0.1));
204204
}
205-
try (var resp = run("METRICS hosts max(rate(request_count)), min(rate(request_count))")) {
205+
try (var resp = run("METRICS hosts | STATS max(rate(request_count)), min(rate(request_count))")) {
206206
assertThat(
207207
resp.columns(),
208208
equalTo(
@@ -218,7 +218,9 @@ record RateKey(String cluster, String host) {
218218
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).max().orElse(0.0), 0.1));
219219
assertThat((double) values.get(0).get(1), closeTo(rates.stream().mapToDouble(d -> d).min().orElse(0.0), 0.1));
220220
}
221-
try (var resp = run("METRICS hosts max(rate(request_count)), avg(rate(request_count)), max(rate(request_count, 1minute))")) {
221+
try (
222+
var resp = run("METRICS hosts | STATS max(rate(request_count)), avg(rate(request_count)), max(rate(request_count, 1minute))")
223+
) {
222224
assertThat(
223225
resp.columns(),
224226
equalTo(
@@ -237,7 +239,7 @@ record RateKey(String cluster, String host) {
237239
assertThat((double) values.get(0).get(1), closeTo(avg, 0.1));
238240
assertThat((double) values.get(0).get(2), closeTo(rates.stream().mapToDouble(d -> d * 60.0).max().orElse(0.0), 0.1));
239241
}
240-
try (var resp = run("METRICS hosts avg(rate(request_count)), avg(rate(request_count, 1second))")) {
242+
try (var resp = run("METRICS hosts | STATS avg(rate(request_count)), avg(rate(request_count, 1second))")) {
241243
assertThat(
242244
resp.columns(),
243245
equalTo(
@@ -254,7 +256,7 @@ record RateKey(String cluster, String host) {
254256
assertThat((double) values.get(0).get(0), closeTo(avg, 0.1));
255257
assertThat((double) values.get(0).get(1), closeTo(avg, 0.1));
256258
}
257-
try (var resp = run("METRICS hosts max(rate(request_count)), min(rate(request_count)), min(cpu), max(cpu)")) {
259+
try (var resp = run("METRICS hosts | STATS max(rate(request_count)), min(rate(request_count)), min(cpu), max(cpu)")) {
258260
assertThat(
259261
resp.columns(),
260262
equalTo(
@@ -294,7 +296,7 @@ record RateKey(String cluster, String host) {
294296
values.add(Objects.requireNonNullElse(rate, 0.0));
295297
}
296298
List<String> sortedKeys = bucketToRates.keySet().stream().sorted().toList();
297-
try (var resp = run("METRICS hosts sum(rate(request_count)) BY cluster | SORT cluster")) {
299+
try (var resp = run("METRICS hosts | STATS sum(rate(request_count)) BY cluster | SORT cluster")) {
298300
assertThat(
299301
resp.columns(),
300302
equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count))", "double"), new ColumnInfoImpl("cluster", "keyword")))
@@ -309,7 +311,7 @@ record RateKey(String cluster, String host) {
309311
assertThat((double) row.get(0), closeTo(bucketToRates.get(key).stream().mapToDouble(d -> d).sum(), 0.1));
310312
}
311313
}
312-
try (var resp = run("METRICS hosts avg(rate(request_count)) BY cluster | SORT cluster")) {
314+
try (var resp = run("METRICS hosts | STATS avg(rate(request_count)) BY cluster | SORT cluster")) {
313315
assertThat(
314316
resp.columns(),
315317
equalTo(List.of(new ColumnInfoImpl("avg(rate(request_count))", "double"), new ColumnInfoImpl("cluster", "keyword")))
@@ -330,7 +332,9 @@ record RateKey(String cluster, String host) {
330332
}
331333
}
332334
}
333-
try (var resp = run("METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY cluster | SORT cluster")) {
335+
try (
336+
var resp = run("METRICS hosts | STATS avg(rate(request_count, 1minute)), avg(rate(request_count)) BY cluster | SORT cluster")
337+
) {
334338
assertThat(
335339
resp.columns(),
336340
equalTo(
@@ -378,7 +382,7 @@ record RateKey(String host, String cluster, long interval) {}
378382
}
379383
}
380384
List<Long> sortedKeys = bucketToRates.keySet().stream().sorted().limit(5).toList();
381-
try (var resp = run("METRICS hosts sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute) | SORT ts | LIMIT 5")) {
385+
try (var resp = run("METRICS hosts | STATS sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute) | SORT ts | LIMIT 5")) {
382386
assertThat(
383387
resp.columns(),
384388
equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count))", "double"), new ColumnInfoImpl("ts", "date")))
@@ -398,7 +402,7 @@ record RateKey(String host, String cluster, long interval) {}
398402
}
399403
}
400404
}
401-
try (var resp = run("METRICS hosts avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute) | SORT ts | LIMIT 5")) {
405+
try (var resp = run("METRICS hosts | STATS avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute) | SORT ts | LIMIT 5")) {
402406
assertThat(
403407
resp.columns(),
404408
equalTo(List.of(new ColumnInfoImpl("avg(rate(request_count))", "double"), new ColumnInfoImpl("ts", "date")))
@@ -420,7 +424,8 @@ record RateKey(String host, String cluster, long interval) {}
420424
}
421425
}
422426
try (var resp = run("""
423-
METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute)
427+
METRICS hosts
428+
| STATS avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute)
424429
| SORT ts
425430
| LIMIT 5
426431
""")) {
@@ -483,7 +488,8 @@ record GroupKey(String cluster, long interval) {}
483488
.limit(5)
484489
.toList();
485490
try (var resp = run("""
486-
METRICS hosts sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute), cluster
491+
METRICS hosts
492+
| STATS sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute), cluster
487493
| SORT ts, cluster
488494
| LIMIT 5""")) {
489495
assertThat(
@@ -513,7 +519,8 @@ METRICS hosts sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute), clust
513519
}
514520
}
515521
try (var resp = run("""
516-
METRICS hosts avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluster
522+
METRICS hosts
523+
| STATS avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluster
517524
| SORT ts, cluster
518525
| LIMIT 5""")) {
519526
assertThat(
@@ -544,7 +551,8 @@ METRICS hosts avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluste
544551
}
545552
}
546553
try (var resp = run("""
547-
METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluster
554+
METRICS hosts
555+
| STATS avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluster
548556
| SORT ts, cluster
549557
| LIMIT 5""")) {
550558
assertThat(
@@ -579,6 +587,7 @@ METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=
579587
}
580588
try (var resp = run("""
581589
METRICS hosts
590+
| STATS
582591
s = sum(rate(request_count)),
583592
c = count(rate(request_count)),
584593
max(rate(request_count)),
@@ -623,7 +632,8 @@ METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=
623632
}
624633
}
625634
try (var resp = run("""
626-
METRICS hosts sum(rate(request_count)), max(cpu) BY ts=bucket(@timestamp, 1 minute), cluster
635+
METRICS hosts
636+
| STATS sum(rate(request_count)), max(cpu) BY ts=bucket(@timestamp, 1 minute), cluster
627637
| SORT ts, cluster
628638
| LIMIT 5""")) {
629639
assertThat(
@@ -660,7 +670,8 @@ METRICS hosts sum(rate(request_count)), max(cpu) BY ts=bucket(@timestamp, 1 minu
660670
}
661671
}
662672
try (var resp = run("""
663-
METRICS hosts sum(rate(request_count)), avg(cpu) BY ts=bucket(@timestamp, 1 minute), cluster
673+
METRICS hosts
674+
| STATS sum(rate(request_count)), avg(cpu) BY ts=bucket(@timestamp, 1 minute), cluster
664675
| SORT ts, cluster
665676
| LIMIT 5""")) {
666677
assertThat(
@@ -715,21 +726,21 @@ record RateKey(String cluster, String host) {
715726
rates.add(v);
716727
}
717728
}
718-
try (var resp = run("METRICS hosts sum(abs(rate(request_count, 1second)))")) {
729+
try (var resp = run("METRICS hosts | STATS sum(abs(rate(request_count, 1second)))")) {
719730
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(abs(rate(request_count, 1second)))", "double"))));
720731
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
721732
assertThat(values, hasSize(1));
722733
assertThat(values.get(0), hasSize(1));
723734
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).sum(), 0.1));
724735
}
725-
try (var resp = run("METRICS hosts sum(10.0 * rate(request_count, 1second))")) {
736+
try (var resp = run("METRICS hosts | STATS sum(10.0 * rate(request_count, 1second))")) {
726737
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(10.0 * rate(request_count, 1second))", "double"))));
727738
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
728739
assertThat(values, hasSize(1));
729740
assertThat(values.get(0), hasSize(1));
730741
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d * 10.0).sum(), 0.1));
731742
}
732-
try (var resp = run("METRICS hosts sum(20 * rate(request_count, 1second) + 10 * floor(rate(request_count, 1second)))")) {
743+
try (var resp = run("METRICS hosts | STATS sum(20 * rate(request_count, 1second) + 10 * floor(rate(request_count, 1second)))")) {
733744
assertThat(
734745
resp.columns(),
735746
equalTo(

‎x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -62,7 +62,6 @@ import ChangePoint,
6262
Fork,
6363
Join,
6464
Lookup,
65-
Metrics,
6665
MvExpand,
6766
Project,
6867
Rrf,

0 commit comments

Comments
 (0)