Skip to content

Update metrics command #125193

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Mar 20, 2025
Merged

Update metrics command #125193

Show file tree
Hide file tree
Changes from 3 commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
4467f60
Update metrics command
dnhatn Mar 18, 2025
3fec912
Fix metrics
dnhatn Mar 19, 2025
ffded6b
Merge remote-tracking branch 'elastic/main' into metrics-command
dnhatn Mar 19, 2025
aa3a26c
PruneUnusedIndexMode
dnhatn Mar 20, 2025
3453f98
Feedback
dnhatn Mar 20, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
50 changes: 25 additions & 25 deletions x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-metrics.csv-spec
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
metricsWithoutAggs
required_capability: metrics_syntax
required_capability: metrics_command
METRICS k8s | sort @timestamp DESC, cluster, pod | keep @timestamp,cluster,pod,network.bytes_in,network.cost | limit 5;

@timestamp:datetime | cluster:keyword | pod: keyword| network.bytes_in:long | network.cost:double
Expand All @@ -11,9 +11,9 @@ METRICS k8s | sort @timestamp DESC, cluster, pod | keep @timestamp,cluster,pod,n
;

metricsWithAggsAndSourceQuoting
required_capability: metrics_syntax
required_capability: metrics_command
required_capability: double_quotes_source_enclosing
METRICS "k8s" max_bytes=max(to_long(network.total_bytes_in)) BY cluster | SORT max_bytes DESC;
METRICS "k8s" | STATS max_bytes=max(to_long(network.total_bytes_in)) BY cluster | SORT max_bytes DESC;

max_bytes:long | cluster: keyword
10797 | qa
Expand All @@ -22,49 +22,49 @@ max_bytes:long | cluster: keyword
;

maxRateAndSourceTripleQuoting
required_capability: metrics_syntax
required_capability: metrics_command
required_capability: double_quotes_source_enclosing
METRICS """k8s""" max(rate(network.total_bytes_in, 1minute));
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute));

max(rate(network.total_bytes_in, 1minute)): double
790.4235090751945
;

maxCost
required_capability: metrics_syntax
METRICS k8s max_cost=max(rate(network.total_cost));
required_capability: metrics_command
METRICS k8s | STATS max_cost=max(rate(network.total_cost));

max_cost: double
0.16151685393258428
;

maxRateAndBytes
required_capability: metrics_syntax
METRICS k8s max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in);
required_capability: metrics_command
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in);

max(rate(network.total_bytes_in, 1minute)): double | max(network.bytes_in): long
790.4235090751945 | 1021
;

`maxRateAndMarkupBytes`
required_capability: metrics_syntax
METRICS k8s max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in * 1.05);
required_capability: metrics_command
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in * 1.05);

max(rate(network.total_bytes_in, 1minute)): double | max(network.bytes_in * 1.05): double
790.4235090751945 | 1072.05
;

maxRateAndBytesAndCost
required_capability: metrics_syntax
METRICS k8s max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in), max(rate(network.total_cost));
required_capability: metrics_command
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in), max(rate(network.total_cost));

max(rate(network.total_bytes_in, 1minute)): double| max(network.bytes_in): long| max(rate(network.total_cost)): double
790.4235090751945 | 1021 | 0.16151685393258428
;

sumRate
required_capability: metrics_syntax
METRICS k8s bytes=sum(rate(network.total_bytes_in)), sum(rate(network.total_cost)) BY cluster | SORT cluster;
required_capability: metrics_command
METRICS k8s | STATS bytes=sum(rate(network.total_bytes_in)), sum(rate(network.total_cost)) BY cluster | SORT cluster;

bytes: double | sum(rate(network.total_cost)): double | cluster: keyword
24.49149357711476 | 0.3018995503437827 | prod
Expand All @@ -73,17 +73,17 @@ bytes: double | sum(rate(network.total_cost)): double | cluster: keyword
;

oneRateWithBucket
required_capability: metrics_syntax
METRICS k8s max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 2;
required_capability: metrics_command
METRICS k8s | STATS max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 2;

max(rate(network.total_bytes_in)): double | time_bucket:date
10.594594594594595 | 2024-05-10T00:20:00.000Z
23.702205882352942 | 2024-05-10T00:15:00.000Z
;

twoRatesWithBucket
required_capability: metrics_syntax
METRICS k8s max(rate(network.total_bytes_in)), sum(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 3;
required_capability: metrics_command
METRICS k8s | STATS max(rate(network.total_bytes_in)), sum(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute) | SORT time_bucket DESC | LIMIT 3;

max(rate(network.total_bytes_in)): double | sum(rate(network.total_bytes_in)): double | time_bucket:date
10.594594594594595 | 42.70864495221802 | 2024-05-10T00:20:00.000Z
Expand All @@ -93,8 +93,8 @@ max(rate(network.total_bytes_in)): double | sum(rate(network.total_bytes_in)): d


oneRateWithBucketAndCluster
required_capability: metrics_syntax
METRICS k8s max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC, cluster | LIMIT 6;
required_capability: metrics_command
METRICS k8s | STATS max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC, cluster | LIMIT 6;

max(rate(network.total_bytes_in)): double | time_bucket:date | cluster: keyword
10.594594594594595 | 2024-05-10T00:20:00.000Z | prod
Expand All @@ -106,8 +106,8 @@ max(rate(network.total_bytes_in)): double | time_bucket:date | cluster:
;

BytesAndCostByBucketAndCluster
required_capability: metrics_syntax
METRICS k8s max(rate(network.total_bytes_in)), max(network.cost) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC, cluster | LIMIT 6;
required_capability: metrics_command
METRICS k8s | STATS max(rate(network.total_bytes_in)), max(network.cost) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC, cluster | LIMIT 6;

max(rate(network.total_bytes_in)): double | max(network.cost): double | time_bucket:date | cluster: keyword
10.594594594594595 | 10.75 | 2024-05-10T00:20:00.000Z | prod
Expand All @@ -119,8 +119,8 @@ max(rate(network.total_bytes_in)): double | max(network.cost): double | time_buc
;

oneRateWithBucketAndClusterThenFilter
required_capability: metrics_syntax
METRICS k8s max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | WHERE cluster=="prod" | SORT time_bucket DESC | LIMIT 3;
required_capability: metrics_command
METRICS k8s | WHERE cluster=="prod" | STATS max(rate(network.total_bytes_in)) BY time_bucket = bucket(@timestamp,5minute), cluster | SORT time_bucket DESC | LIMIT 3;

max(rate(network.total_bytes_in)): double | time_bucket:date | cluster: keyword
10.594594594594595 | 2024-05-10T00:20:00.000Z | prod
Expand Down
6 changes: 4 additions & 2 deletions ...in/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TelemetryIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -155,8 +155,10 @@ public static Iterable<Object[]> parameters() {
) },
new Object[] {
new Test(
"METRICS idx max(id) BY host | LIMIT 10",
Build.current().isSnapshot() ? Map.ofEntries(Map.entry("METRICS", 1), Map.entry("LIMIT", 1)) : Collections.emptyMap(),
"METRICS idx | STATS max(id) BY host | LIMIT 10",
Build.current().isSnapshot()
? Map.ofEntries(Map.entry("METRICS", 1), Map.entry("STATS", 1), Map.entry("LIMIT", 1))
: Collections.emptyMap(),
Build.current().isSnapshot() ? Map.ofEntries(Map.entry("MAX", 1)) : Collections.emptyMap(),
Build.current().isSnapshot()
) }
Expand Down
51 changes: 31 additions & 20 deletions ...n/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -148,7 +148,7 @@ public void populateIndex() {
public void testSimpleMetrics() {
List<String> sortedGroups = docs.stream().map(d -> d.host).distinct().sorted().toList();
client().admin().indices().prepareRefresh("hosts").get();
try (EsqlQueryResponse resp = run("METRICS hosts load=avg(cpu) BY host | SORT host")) {
try (EsqlQueryResponse resp = run("METRICS hosts | STATS load=avg(cpu) BY host | SORT host")) {
List<List<Object>> rows = EsqlTestUtils.getValuesList(resp);
assertThat(rows, hasSize(sortedGroups.size()));
for (int i = 0; i < rows.size(); i++) {
Expand Down Expand Up @@ -195,14 +195,14 @@ record RateKey(String cluster, String host) {
rates.add(v);
}
}
try (var resp = run("METRICS hosts sum(rate(request_count, 1second))")) {
try (var resp = run("METRICS hosts | STATS sum(rate(request_count, 1second))")) {
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count, 1second))", "double"))));
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
assertThat(values, hasSize(1));
assertThat(values.get(0), hasSize(1));
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).sum(), 0.1));
}
try (var resp = run("METRICS hosts max(rate(request_count)), min(rate(request_count))")) {
try (var resp = run("METRICS hosts | STATS max(rate(request_count)), min(rate(request_count))")) {
assertThat(
resp.columns(),
equalTo(
Expand All @@ -218,7 +218,9 @@ record RateKey(String cluster, String host) {
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).max().orElse(0.0), 0.1));
assertThat((double) values.get(0).get(1), closeTo(rates.stream().mapToDouble(d -> d).min().orElse(0.0), 0.1));
}
try (var resp = run("METRICS hosts max(rate(request_count)), avg(rate(request_count)), max(rate(request_count, 1minute))")) {
try (
var resp = run("METRICS hosts | STATS max(rate(request_count)), avg(rate(request_count)), max(rate(request_count, 1minute))")
) {
assertThat(
resp.columns(),
equalTo(
Expand All @@ -237,7 +239,7 @@ record RateKey(String cluster, String host) {
assertThat((double) values.get(0).get(1), closeTo(avg, 0.1));
assertThat((double) values.get(0).get(2), closeTo(rates.stream().mapToDouble(d -> d * 60.0).max().orElse(0.0), 0.1));
}
try (var resp = run("METRICS hosts avg(rate(request_count)), avg(rate(request_count, 1second))")) {
try (var resp = run("METRICS hosts | STATS avg(rate(request_count)), avg(rate(request_count, 1second))")) {
assertThat(
resp.columns(),
equalTo(
Expand All @@ -254,7 +256,7 @@ record RateKey(String cluster, String host) {
assertThat((double) values.get(0).get(0), closeTo(avg, 0.1));
assertThat((double) values.get(0).get(1), closeTo(avg, 0.1));
}
try (var resp = run("METRICS hosts max(rate(request_count)), min(rate(request_count)), min(cpu), max(cpu)")) {
try (var resp = run("METRICS hosts | STATS max(rate(request_count)), min(rate(request_count)), min(cpu), max(cpu)")) {
assertThat(
resp.columns(),
equalTo(
Expand Down Expand Up @@ -294,7 +296,7 @@ record RateKey(String cluster, String host) {
values.add(Objects.requireNonNullElse(rate, 0.0));
}
List<String> sortedKeys = bucketToRates.keySet().stream().sorted().toList();
try (var resp = run("METRICS hosts sum(rate(request_count)) BY cluster | SORT cluster")) {
try (var resp = run("METRICS hosts | STATS sum(rate(request_count)) BY cluster | SORT cluster")) {
assertThat(
resp.columns(),
equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count))", "double"), new ColumnInfoImpl("cluster", "keyword")))
Expand All @@ -309,7 +311,7 @@ record RateKey(String cluster, String host) {
assertThat((double) row.get(0), closeTo(bucketToRates.get(key).stream().mapToDouble(d -> d).sum(), 0.1));
}
}
try (var resp = run("METRICS hosts avg(rate(request_count)) BY cluster | SORT cluster")) {
try (var resp = run("METRICS hosts | STATS avg(rate(request_count)) BY cluster | SORT cluster")) {
assertThat(
resp.columns(),
equalTo(List.of(new ColumnInfoImpl("avg(rate(request_count))", "double"), new ColumnInfoImpl("cluster", "keyword")))
Expand All @@ -330,7 +332,9 @@ record RateKey(String cluster, String host) {
}
}
}
try (var resp = run("METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY cluster | SORT cluster")) {
try (
var resp = run("METRICS hosts | STATS avg(rate(request_count, 1minute)), avg(rate(request_count)) BY cluster | SORT cluster")
) {
assertThat(
resp.columns(),
equalTo(
Expand Down Expand Up @@ -378,7 +382,7 @@ record RateKey(String host, String cluster, long interval) {}
}
}
List<Long> sortedKeys = bucketToRates.keySet().stream().sorted().limit(5).toList();
try (var resp = run("METRICS hosts sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute) | SORT ts | LIMIT 5")) {
try (var resp = run("METRICS hosts | STATS sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute) | SORT ts | LIMIT 5")) {
assertThat(
resp.columns(),
equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count))", "double"), new ColumnInfoImpl("ts", "date")))
Expand All @@ -398,7 +402,7 @@ record RateKey(String host, String cluster, long interval) {}
}
}
}
try (var resp = run("METRICS hosts avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute) | SORT ts | LIMIT 5")) {
try (var resp = run("METRICS hosts | STATS avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute) | SORT ts | LIMIT 5")) {
assertThat(
resp.columns(),
equalTo(List.of(new ColumnInfoImpl("avg(rate(request_count))", "double"), new ColumnInfoImpl("ts", "date")))
Expand All @@ -420,7 +424,8 @@ record RateKey(String host, String cluster, long interval) {}
}
}
try (var resp = run("""
METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute)
METRICS hosts
| STATS avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute)
| SORT ts
| LIMIT 5
""")) {
Expand Down Expand Up @@ -483,7 +488,8 @@ record GroupKey(String cluster, long interval) {}
.limit(5)
.toList();
try (var resp = run("""
METRICS hosts sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute), cluster
METRICS hosts
| STATS sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute), cluster
| SORT ts, cluster
| LIMIT 5""")) {
assertThat(
Expand Down Expand Up @@ -513,7 +519,8 @@ METRICS hosts sum(rate(request_count)) BY ts=bucket(@timestamp, 1 minute), clust
}
}
try (var resp = run("""
METRICS hosts avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluster
METRICS hosts
| STATS avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluster
| SORT ts, cluster
| LIMIT 5""")) {
assertThat(
Expand Down Expand Up @@ -544,7 +551,8 @@ METRICS hosts avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluste
}
}
try (var resp = run("""
METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluster
METRICS hosts
| STATS avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluster
| SORT ts, cluster
| LIMIT 5""")) {
assertThat(
Expand Down Expand Up @@ -579,6 +587,7 @@ METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=
}
try (var resp = run("""
METRICS hosts
| STATS
s = sum(rate(request_count)),
c = count(rate(request_count)),
max(rate(request_count)),
Expand Down Expand Up @@ -623,7 +632,8 @@ METRICS hosts avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=
}
}
try (var resp = run("""
METRICS hosts sum(rate(request_count)), max(cpu) BY ts=bucket(@timestamp, 1 minute), cluster
METRICS hosts
| STATS sum(rate(request_count)), max(cpu) BY ts=bucket(@timestamp, 1 minute), cluster
| SORT ts, cluster
| LIMIT 5""")) {
assertThat(
Expand Down Expand Up @@ -660,7 +670,8 @@ METRICS hosts sum(rate(request_count)), max(cpu) BY ts=bucket(@timestamp, 1 minu
}
}
try (var resp = run("""
METRICS hosts sum(rate(request_count)), avg(cpu) BY ts=bucket(@timestamp, 1 minute), cluster
METRICS hosts
| STATS sum(rate(request_count)), avg(cpu) BY ts=bucket(@timestamp, 1 minute), cluster
| SORT ts, cluster
| LIMIT 5""")) {
assertThat(
Expand Down Expand Up @@ -715,21 +726,21 @@ record RateKey(String cluster, String host) {
rates.add(v);
}
}
try (var resp = run("METRICS hosts sum(abs(rate(request_count, 1second)))")) {
try (var resp = run("METRICS hosts | STATS sum(abs(rate(request_count, 1second)))")) {
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(abs(rate(request_count, 1second)))", "double"))));
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
assertThat(values, hasSize(1));
assertThat(values.get(0), hasSize(1));
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).sum(), 0.1));
}
try (var resp = run("METRICS hosts sum(10.0 * rate(request_count, 1second))")) {
try (var resp = run("METRICS hosts | STATS sum(10.0 * rate(request_count, 1second))")) {
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(10.0 * rate(request_count, 1second))", "double"))));
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
assertThat(values, hasSize(1));
assertThat(values.get(0), hasSize(1));
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d * 10.0).sum(), 0.1));
}
try (var resp = run("METRICS hosts sum(20 * rate(request_count, 1second) + 10 * floor(rate(request_count, 1second)))")) {
try (var resp = run("METRICS hosts | STATS sum(20 * rate(request_count, 1second) + 10 * floor(rate(request_count, 1second)))")) {
assertThat(
resp.columns(),
equalTo(
Expand Down
1 change: 0 additions & 1 deletion x-pack/plugin/esql/src/main/antlr/EsqlBaseLexer.g4
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@ import ChangePoint,
Fork,
Join,
Lookup,
Metrics,
MvExpand,
Project,
Rrf,
Expand Down
Loading
Loading