Skip to content

Commit 271739e

Browse files
dnhatndnhatn
committed
Add bucket interval time-series rate function
1 parent 61d92d6 commit 271739e

File tree

9 files changed

+111
-218
lines changed

9 files changed

+111
-218
lines changed

‎x-pack/plugin/esql/qa/server/mixed-cluster/src/javaRestTest/java/org/elasticsearch/xpack/esql/qa/mixed/MixedClusterEsqlSpecIT.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,11 @@ protected void shouldSkipTest(String testName) throws IOException {
7070
assumeTrue("Test " + testName + " is skipped on " + bwcVersion, isEnabled(testName, instructions, bwcVersion));
7171
}
7272

73+
@Override
74+
protected boolean supportTimeSeries() {
75+
return false;
76+
}
77+
7378
@Override
7479
protected boolean enableRoundingDoubleValuesOnAsserting() {
7580
return true;

‎x-pack/plugin/esql/qa/server/multi-clusters/src/javaRestTest/java/org/elasticsearch/xpack/esql/ccq/MultiClusterSpecIT.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -132,6 +132,11 @@ protected void shouldSkipTest(String testName) throws IOException {
132132
assumeFalse("UNMAPPED FIELDS not yet supported in CCS", testCase.requiredCapabilities.contains(UNMAPPED_FIELDS.capabilityName()));
133133
}
134134

135+
@Override
136+
protected boolean supportTimeSeries() {
137+
return false;
138+
}
139+
135140
private TestFeatureService remoteFeaturesService() throws IOException {
136141
if (remoteFeaturesService == null) {
137142
var remoteNodeVersions = readVersionsFromNodesInfo(remoteClusterClient());

‎x-pack/plugin/esql/qa/server/src/main/java/org/elasticsearch/xpack/esql/qa/rest/EsqlSpecTestCase.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -173,6 +173,9 @@ protected void shouldSkipTest(String testName) throws IOException {
173173
|| testCase.requiredCapabilities.contains("semantic_text_field_caps")) {
174174
assumeTrue("Inference test service needs to be supported for semantic_text", supportsInferenceTestService());
175175
}
176+
if (testCase.query.startsWith("METRICS ") || testCase.query.startsWith("metrics ")) {
177+
assumeTrue("Time-series tests are disabled in mixed clusters until the development stabilizes", supportTimeSeries());
178+
}
176179
checkCapabilities(adminClient(), testFeatureService, testName, testCase);
177180
assumeTrue("Test " + testName + " is not enabled", isEnabled(testName, instructions, Version.CURRENT));
178181
if (supportsSourceFieldMapping() == false) {
@@ -227,6 +230,10 @@ public static boolean hasCapabilities(RestClient client, List<String> requiredCa
227230
return false;
228231
}
229232

233+
protected boolean supportTimeSeries() {
234+
return true;
235+
}
236+
230237
protected boolean supportsInferenceTestService() {
231238
return true;
232239
}

‎x-pack/plugin/esql/qa/testFixtures/src/main/resources/k8s-metrics.csv-spec‎

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -40,26 +40,26 @@ max_cost: double
4040

4141
maxRateAndBytes
4242
required_capability: metrics_command
43-
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in);
43+
METRICS k8s | STATS max(rate(network.total_bytes_in)), max(network.bytes_in);
4444

45-
max(rate(network.total_bytes_in, 1minute)): double | max(network.bytes_in): long
46-
790.4235090751945 | 1021
45+
max(rate(network.total_bytes_in)): double | max(network.bytes_in): long
46+
13.17372515125324 | 1021
4747
;
4848

4949
`maxRateAndMarkupBytes`
5050
required_capability: metrics_command
51-
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in * 1.05);
51+
METRICS k8s | STATS max(rate(network.total_bytes_in)), max(network.bytes_in * 1.05);
5252

53-
max(rate(network.total_bytes_in, 1minute)): double | max(network.bytes_in * 1.05): double
54-
790.4235090751945 | 1072.05
53+
max(rate(network.total_bytes_in)): double | max(network.bytes_in * 1.05): double
54+
13.17372515125324 | 1072.05
5555
;
5656

5757
maxRateAndBytesAndCost
5858
required_capability: metrics_command
5959
METRICS k8s | STATS max(rate(network.total_bytes_in, 1minute)), max(network.bytes_in), max(rate(network.total_cost));
6060

61-
max(rate(network.total_bytes_in, 1minute)): double| max(network.bytes_in): long| max(rate(network.total_cost)): double
62-
790.4235090751945 | 1021 | 0.16151685393258428
61+
max(rate(network.total_bytes_in)): double| max(network.bytes_in): long| max(rate(network.total_cost)): double
62+
13.17372515125324 | 1021 | 0.16151685393258428
6363
;
6464

6565
sumRate

‎x-pack/plugin/esql/src/internalClusterTest/java/org/elasticsearch/xpack/esql/action/TimeSeriesIT.java‎

Lines changed: 16 additions & 122 deletions
Original file line numberDiff line numberDiff line change
@@ -195,8 +195,8 @@ record RateKey(String cluster, String host) {
195195
rates.add(v);
196196
}
197197
}
198-
try (var resp = run("METRICS hosts | STATS sum(rate(request_count, 1second))")) {
199-
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count, 1second))", "double", null))));
198+
try (var resp = run("METRICS hosts | STATS sum(rate(request_count))")) {
199+
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(rate(request_count))", "double", null))));
200200
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
201201
assertThat(values, hasSize(1));
202202
assertThat(values.get(0), hasSize(1));
@@ -218,42 +218,21 @@ record RateKey(String cluster, String host) {
218218
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).max().orElse(0.0), 0.1));
219219
assertThat((double) values.get(0).get(1), closeTo(rates.stream().mapToDouble(d -> d).min().orElse(0.0), 0.1));
220220
}
221-
try (
222-
var resp = run("METRICS hosts | STATS max(rate(request_count)), avg(rate(request_count)), max(rate(request_count, 1minute))")
223-
) {
221+
try (var resp = run("METRICS hosts | STATS max(rate(request_count)), avg(rate(request_count))")) {
224222
assertThat(
225223
resp.columns(),
226224
equalTo(
227225
List.of(
228226
new ColumnInfoImpl("max(rate(request_count))", "double", null),
229-
new ColumnInfoImpl("avg(rate(request_count))", "double", null),
230-
new ColumnInfoImpl("max(rate(request_count, 1minute))", "double", null)
231-
)
232-
)
233-
);
234-
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
235-
assertThat(values, hasSize(1));
236-
assertThat(values.get(0), hasSize(3));
237-
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).max().orElse(0.0), 0.1));
238-
final double avg = rates.isEmpty() ? 0.0 : rates.stream().mapToDouble(d -> d).sum() / rates.size();
239-
assertThat((double) values.get(0).get(1), closeTo(avg, 0.1));
240-
assertThat((double) values.get(0).get(2), closeTo(rates.stream().mapToDouble(d -> d * 60.0).max().orElse(0.0), 0.1));
241-
}
242-
try (var resp = run("METRICS hosts | STATS avg(rate(request_count)), avg(rate(request_count, 1second))")) {
243-
assertThat(
244-
resp.columns(),
245-
equalTo(
246-
List.of(
247-
new ColumnInfoImpl("avg(rate(request_count))", "double", null),
248-
new ColumnInfoImpl("avg(rate(request_count, 1second))", "double", null)
227+
new ColumnInfoImpl("avg(rate(request_count))", "double", null)
249228
)
250229
)
251230
);
252231
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
253232
assertThat(values, hasSize(1));
254233
assertThat(values.get(0), hasSize(2));
234+
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).max().orElse(0.0), 0.1));
255235
final double avg = rates.isEmpty() ? 0.0 : rates.stream().mapToDouble(d -> d).sum() / rates.size();
256-
assertThat((double) values.get(0).get(0), closeTo(avg, 0.1));
257236
assertThat((double) values.get(0).get(1), closeTo(avg, 0.1));
258237
}
259238
try (var resp = run("METRICS hosts | STATS max(rate(request_count)), min(rate(request_count)), min(cpu), max(cpu)")) {
@@ -336,34 +315,26 @@ record RateKey(String cluster, String host) {
336315
}
337316
}
338317
}
339-
try (
340-
var resp = run("METRICS hosts | STATS avg(rate(request_count, 1minute)), avg(rate(request_count)) BY cluster | SORT cluster")
341-
) {
318+
try (var resp = run("METRICS hosts | STATS avg(rate(request_count)) BY cluster | SORT cluster")) {
342319
assertThat(
343320
resp.columns(),
344321
equalTo(
345-
List.of(
346-
new ColumnInfoImpl("avg(rate(request_count, 1minute))", "double", null),
347-
new ColumnInfoImpl("avg(rate(request_count))", "double", null),
348-
new ColumnInfoImpl("cluster", "keyword", null)
349-
)
322+
List.of(new ColumnInfoImpl("avg(rate(request_count))", "double", null), new ColumnInfoImpl("cluster", "keyword", null))
350323
)
351324
);
352325
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
353326
assertThat(values, hasSize(bucketToRates.size()));
354327
for (int i = 0; i < bucketToRates.size(); i++) {
355328
List<Object> row = values.get(i);
356-
assertThat(row, hasSize(3));
329+
assertThat(row, hasSize(2));
357330
String key = sortedKeys.get(i);
358-
assertThat(row.get(2), equalTo(key));
331+
assertThat(row.get(1), equalTo(key));
359332
List<Double> rates = bucketToRates.get(key);
360333
if (rates.isEmpty()) {
361334
assertThat(row.get(0), equalTo(0.0));
362-
assertThat(row.get(1), equalTo(0.0));
363335
} else {
364336
double avg = rates.stream().mapToDouble(d -> d).sum() / rates.size();
365-
assertThat((double) row.get(0), closeTo(avg * 60.0f, 0.1));
366-
assertThat((double) row.get(1), closeTo(avg, 0.1));
337+
assertThat((double) row.get(0), closeTo(avg, 0.1));
367338
}
368339
}
369340
}
@@ -427,40 +398,6 @@ record RateKey(String host, String cluster, long interval) {}
427398
}
428399
}
429400
}
430-
try (var resp = run("""
431-
METRICS hosts
432-
| STATS avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute)
433-
| SORT ts
434-
| LIMIT 5
435-
""")) {
436-
assertThat(
437-
resp.columns(),
438-
equalTo(
439-
List.of(
440-
new ColumnInfoImpl("avg(rate(request_count, 1minute))", "double", null),
441-
new ColumnInfoImpl("avg(rate(request_count))", "double", null),
442-
new ColumnInfoImpl("ts", "date", null)
443-
)
444-
)
445-
);
446-
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
447-
assertThat(values, hasSize(sortedKeys.size()));
448-
for (int i = 0; i < sortedKeys.size(); i++) {
449-
List<Object> row = values.get(i);
450-
assertThat(row, hasSize(3));
451-
long key = sortedKeys.get(i);
452-
assertThat(row.get(2), equalTo(DEFAULT_DATE_TIME_FORMATTER.formatMillis(key)));
453-
List<Double> bucketValues = bucketToRates.get(key);
454-
if (bucketValues.isEmpty()) {
455-
assertNull(row.get(0));
456-
assertNull(row.get(1));
457-
} else {
458-
double avg = bucketValues.stream().mapToDouble(d -> d).sum() / bucketValues.size();
459-
assertThat((double) row.get(0), closeTo(avg * 60.0f, 0.1));
460-
assertThat((double) row.get(1), closeTo(avg, 0.1));
461-
}
462-
}
463-
}
464401
}
465402

466403
public void testRateWithTimeBucketAndCluster() {
@@ -554,41 +491,6 @@ record GroupKey(String cluster, long interval) {}
554491
}
555492
}
556493
}
557-
try (var resp = run("""
558-
METRICS hosts
559-
| STATS avg(rate(request_count, 1minute)), avg(rate(request_count)) BY ts=bucket(@timestamp, 1minute), cluster
560-
| SORT ts, cluster
561-
| LIMIT 5""")) {
562-
assertThat(
563-
resp.columns(),
564-
equalTo(
565-
List.of(
566-
new ColumnInfoImpl("avg(rate(request_count, 1minute))", "double", null),
567-
new ColumnInfoImpl("avg(rate(request_count))", "double", null),
568-
new ColumnInfoImpl("ts", "date", null),
569-
new ColumnInfoImpl("cluster", "keyword", null)
570-
)
571-
)
572-
);
573-
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
574-
assertThat(values, hasSize(sortedKeys.size()));
575-
for (int i = 0; i < sortedKeys.size(); i++) {
576-
List<Object> row = values.get(i);
577-
assertThat(row, hasSize(4));
578-
var key = sortedKeys.get(i);
579-
assertThat(row.get(2), equalTo(DEFAULT_DATE_TIME_FORMATTER.formatMillis(key.interval)));
580-
assertThat(row.get(3), equalTo(key.cluster));
581-
List<Double> bucketValues = rateBuckets.get(key);
582-
if (bucketValues.isEmpty()) {
583-
assertNull(row.get(0));
584-
assertNull(row.get(1));
585-
} else {
586-
double avg = bucketValues.stream().mapToDouble(d -> d).sum() / bucketValues.size();
587-
assertThat((double) row.get(0), closeTo(avg * 60.0f, 0.1));
588-
assertThat((double) row.get(1), closeTo(avg, 0.1));
589-
}
590-
}
591-
}
592494
try (var resp = run("""
593495
METRICS hosts
594496
| STATS
@@ -730,32 +632,24 @@ record RateKey(String cluster, String host) {
730632
rates.add(v);
731633
}
732634
}
733-
try (var resp = run("METRICS hosts | STATS sum(abs(rate(request_count, 1second)))")) {
734-
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(abs(rate(request_count, 1second)))", "double", null))));
635+
try (var resp = run("METRICS hosts | STATS sum(abs(rate(request_count)))")) {
636+
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(abs(rate(request_count)))", "double", null))));
735637
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
736638
assertThat(values, hasSize(1));
737639
assertThat(values.get(0), hasSize(1));
738640
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d).sum(), 0.1));
739641
}
740-
try (var resp = run("METRICS hosts | STATS sum(10.0 * rate(request_count, 1second))")) {
741-
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(10.0 * rate(request_count, 1second))", "double", null))));
642+
try (var resp = run("METRICS hosts | STATS sum(10.0 * rate(request_count))")) {
643+
assertThat(resp.columns(), equalTo(List.of(new ColumnInfoImpl("sum(10.0 * rate(request_count))", "double", null))));
742644
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
743645
assertThat(values, hasSize(1));
744646
assertThat(values.get(0), hasSize(1));
745647
assertThat((double) values.get(0).get(0), closeTo(rates.stream().mapToDouble(d -> d * 10.0).sum(), 0.1));
746648
}
747-
try (var resp = run("METRICS hosts | STATS sum(20 * rate(request_count, 1second) + 10 * floor(rate(request_count, 1second)))")) {
649+
try (var resp = run("METRICS hosts | STATS sum(20 * rate(request_count) + 10 * floor(rate(request_count)))")) {
748650
assertThat(
749651
resp.columns(),
750-
equalTo(
751-
List.of(
752-
new ColumnInfoImpl(
753-
"sum(20 * rate(request_count, 1second) + 10 * floor(rate(request_count, 1second)))",
754-
"double",
755-
null
756-
)
757-
)
758-
)
652+
equalTo(List.of(new ColumnInfoImpl("sum(20 * rate(request_count) + 10 * floor(rate(request_count)))", "double", null)))
759653
);
760654
List<List<Object>> values = EsqlTestUtils.getValuesList(resp);
761655
assertThat(values, hasSize(1));

0 commit comments

Comments
 (0)