Skip to content

Commit 0a0890c

Browse files
wankai123wankai123
authored
MQE: Support top_n_of function for merging multiple metrics topn query. (#12937)
1 parent c6ef231 commit 0a0890c

File tree

9 files changed

+186
-8
lines changed

9 files changed

+186
-8
lines changed

‎docs/en/api/metrics-query-expression.md‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -217,6 +217,7 @@ round(service_cpm / 60 , 2)
217217
The different operators could impact the `ExpressionResultType`, please refer to the above table.
218218

219219
## TopN Operation
220+
### TopN Query
220221
TopN Operation takes an expression and performs calculation to get the TopN of Services/Instances/Endpoints.
221222
The result depends on the `entity` condition in the query.
222223
- Global TopN:
@@ -264,6 +265,25 @@ top_n(service_instance_cpm, 10, des)
264265
### Result Type
265266
According to the type of the metric, the `ExpressionResultType` of the expression will be `SORTED_LIST` or `RECORD_LIST`.
266267

268+
### Multiple TopNs Merging
269+
As the difference between agent and ebpf, some metrics would be separated, e.g. service cpm and k8s service cpm.
270+
If you want to merge the topN results of these metrics, you can use the `ton_n_of` operation to merge the results.
271+
272+
expression:
273+
```text
274+
ton_n_of(<top_n>, <top_n>, ...,<top_number>, <order>)
275+
```
276+
277+
- `<top_n>` is the [topN](#topn-query) expression. The result type of those tonN expression should be same, can be `SORTED_LIST` or `RECORD_LIST`, `but can not be mixed`.
278+
- `<top_number>` is the number of the merged top results, should be a positive integer.
279+
- `<order>` is the order of the merged top results. The value of `<order>` can be `asc` or `des`.
280+
281+
for example:
282+
If we want to get the top 10 services with the highest `service_cpm` and `k8s_service_cpm`, we can use the following expression:
283+
```text
284+
ton_n_of(top_n(service_cpm, 10, des), top_n(k8s_service_cpm, 10, des), 10, des)
285+
```
286+
267287
## Relabel Operation
268288
Relabel Operation takes an expression and replaces the label with new label on its results.
269289
Since v10.0.0, SkyWalking supports relabel multiple labels.

‎docs/en/changes/changes.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
* Add `bydb.dependencies.properties` config file to define server dependency versions.
5454
* Fix `AvgHistogramPercentileFunction` doesn't have proper field definition for `ranks`.
5555
* BanyanDB: Support the new Property data module.
56+
* MQE: Support `top_n_of` function for merging multiple metrics topn query.
5657

5758
#### UI
5859

‎oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQELexer.g4‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@ RATE: 'rate';
6464

6565
// TopN
6666
TOP_N: 'top_n';
67+
TOP_N_OF: 'top_n_of';
6768

6869
// ViewAsSeq
6970
VIEW_AS_SEQ: 'view_as_seq';

‎oap-server/mqe-grammar/src/main/antlr4/org/apache/skywalking/mqe/rt/grammar/MQEParser.g4‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,8 @@ expression
3333
| mathematical_operator1 L_PAREN expression COMMA parameter R_PAREN #mathematicalOperator1OP
3434
| trend L_PAREN metric COMMA INTEGER R_PAREN #trendOP
3535
| logical_operator L_PAREN expressionList R_PAREN #logicalOperatorOP
36-
| topN L_PAREN metric COMMA INTEGER COMMA order (COMMA attributeList)? R_PAREN #topNOP
36+
| topN #topNOP
37+
| topNOf L_PAREN topN (COMMA topN)* COMMA INTEGER COMMA order R_PAREN #topNOfOP
3738
| relabels L_PAREN expression COMMA label COMMA replaceLabel R_PAREN #relablesOP
3839
| aggregateLabels L_PAREN expression COMMA aggregateLabelsFunc R_PAREN #aggregateLabelsOp
3940
| sort_values L_PAREN expression (COMMA INTEGER)? COMMA order R_PAREN #sortValuesOP
@@ -75,7 +76,8 @@ mathematical_operator1:
7576
trend:
7677
INCREASE | RATE;
7778

78-
topN: TOP_N;
79+
topN: TOP_N L_PAREN metric COMMA INTEGER COMMA order (COMMA attributeList)? R_PAREN;
80+
topNOf: TOP_N_OF;
7981

8082
logical_operator:
8183
VIEW_AS_SEQ | IS_PRESENT;

‎oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/MQEVisitorBase.java‎

Lines changed: 22 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.skywalking.mqe.rt.operation.MathematicalFunctionOp;
3535
import org.apache.skywalking.mqe.rt.operation.SortLabelValuesOp;
3636
import org.apache.skywalking.mqe.rt.operation.SortValuesOp;
37+
import org.apache.skywalking.mqe.rt.operation.TopNOfOp;
3738
import org.apache.skywalking.mqe.rt.operation.TrendOp;
3839
import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResult;
3940
import org.apache.skywalking.mqe.rt.exception.IllegalExpressionException;
@@ -240,7 +241,27 @@ public ExpressionResult visitTopNOP(MQEParser.TopNOPContext ctx) {
240241
DebuggingTraceContext traceContext = TRACE_CONTEXT.get();
241242
DebuggingSpan span = traceContext.createSpan("MQE TopN OP: " + ctx.getText());
242243
try {
243-
return visit(ctx.metric());
244+
return visit(ctx.topN().metric());
245+
} finally {
246+
traceContext.stopSpan(span);
247+
}
248+
}
249+
250+
@Override
251+
public ExpressionResult visitTopNOfOP(MQEParser.TopNOfOPContext ctx) {
252+
DebuggingTraceContext traceContext = TRACE_CONTEXT.get();
253+
DebuggingSpan span = traceContext.createSpan("MQE TopNOf OP: " + ctx.getText());
254+
try {
255+
List<MQEParser.TopNContext> topNContexts = ctx.topN();
256+
List<ExpressionResult> topNResults = new ArrayList<>();
257+
for (MQEParser.TopNContext topNContext : topNContexts) {
258+
topNResults.add(visit(topNContext.metric()));
259+
}
260+
try {
261+
return TopNOfOp.doMergeTopNResult(topNResults, Integer.parseInt(ctx.INTEGER().getText()), ctx.order().getStart().getType());
262+
} catch (IllegalExpressionException e) {
263+
return getErrorResult(e.getMessage());
264+
}
244265
} finally {
245266
traceContext.stopSpan(span);
246267
}

‎oap-server/mqe-rt/src/main/java/org/apache/skywalking/mqe/rt/operation/TopNOfOp.java‎

Lines changed: 71 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,71 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package org.apache.skywalking.mqe.rt.operation;
20+
21+
import java.util.ArrayList;
22+
import java.util.Comparator;
23+
import java.util.List;
24+
import java.util.stream.Collectors;
25+
import org.apache.skywalking.mqe.rt.exception.IllegalExpressionException;
26+
import org.apache.skywalking.mqe.rt.grammar.MQEParser;
27+
import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResult;
28+
import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResultType;
29+
import org.apache.skywalking.oap.server.core.query.mqe.MQEValue;
30+
import org.apache.skywalking.oap.server.core.query.mqe.MQEValues;
31+
import org.apache.skywalking.oap.server.library.util.StringUtil;
32+
33+
public class TopNOfOp {
34+
public static ExpressionResult doMergeTopNResult(List<ExpressionResult> topNResults,
35+
int limit,
36+
int order) throws IllegalExpressionException {
37+
ExpressionResultType type = null;
38+
List<MQEValue> allValues = new ArrayList<>();
39+
for (ExpressionResult topNResult : topNResults) {
40+
if (StringUtil.isNotEmpty(topNResult.getError())) {
41+
return topNResult;
42+
}
43+
// check the type of topNResults
44+
if (type != null && type != topNResult.getType()) {
45+
throw new IllegalExpressionException("TopN type is not consistent, one is " + type + ", another is " +
46+
topNResult.getType());
47+
}
48+
type = topNResult.getType();
49+
// topN result should have values without label
50+
allValues.addAll(topNResult.getResults().get(0).getValues());
51+
}
52+
if (limit > allValues.size()) {
53+
limit = allValues.size();
54+
}
55+
List<MQEValue> mergedValues = allValues.stream()
56+
// Filter out empty values
57+
.filter(mqeValue -> !mqeValue.isEmptyValue())
58+
.sorted(MQEParser.ASC == order ? Comparator.comparingDouble(
59+
MQEValue::getDoubleValue) :
60+
Comparator.comparingDouble(MQEValue::getDoubleValue)
61+
.reversed())
62+
.limit(limit).collect(Collectors.toList());
63+
64+
ExpressionResult result = new ExpressionResult();
65+
MQEValues mqeValues = new MQEValues();
66+
mqeValues.setValues(mergedValues);
67+
result.getResults().add(mqeValues);
68+
result.setType(type);
69+
return result;
70+
}
71+
}

‎oap-server/mqe-rt/src/test/java/org/apache/skywalking/mqe/rt/TopNOfOpTest.java‎

Lines changed: 59 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,59 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package org.apache.skywalking.mqe.rt;
20+
21+
import java.util.List;
22+
import org.apache.skywalking.mqe.rt.exception.IllegalExpressionException;
23+
import org.apache.skywalking.mqe.rt.grammar.MQEParser;
24+
import org.apache.skywalking.mqe.rt.operation.TopNOfOp;
25+
import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResult;
26+
import org.apache.skywalking.oap.server.core.query.mqe.ExpressionResultType;
27+
import org.junit.jupiter.api.Assertions;
28+
import org.junit.jupiter.api.Test;
29+
30+
public class TopNOfOpTest {
31+
32+
@Test
33+
public void mergeTopNResultTest() throws IllegalExpressionException {
34+
MockData mockData = new MockData();
35+
List<ExpressionResult> topNResults = List.of(
36+
mockData.newListResult(1000, 100),
37+
mockData.newListResult(600, 500),
38+
mockData.newListResult(300, 2000)
39+
);
40+
ExpressionResult topNResult = TopNOfOp.doMergeTopNResult(topNResults, 2, MQEParser.DES);
41+
Assertions.assertEquals(ExpressionResultType.SORTED_LIST, topNResult.getType());
42+
Assertions.assertEquals(2, topNResult.getResults().get(0).getValues().size());
43+
Assertions.assertEquals(2000, topNResult.getResults().get(0).getValues().get(0).getDoubleValue());
44+
Assertions.assertEquals("service_B", topNResult.getResults().get(0).getValues().get(0).getId());
45+
Assertions.assertEquals(1000, topNResult.getResults().get(0).getValues().get(1).getDoubleValue());
46+
Assertions.assertEquals("service_A", topNResult.getResults().get(0).getValues().get(1).getId());
47+
48+
ExpressionResult topNResultAsc = TopNOfOp.doMergeTopNResult(topNResults, 8, MQEParser.ASC);
49+
Assertions.assertEquals(6, topNResultAsc.getResults().get(0).getValues().size());
50+
Assertions.assertEquals(100, topNResultAsc.getResults().get(0).getValues().get(0).getDoubleValue());
51+
Assertions.assertEquals("service_B", topNResultAsc.getResults().get(0).getValues().get(0).getId());
52+
Assertions.assertEquals(2000, topNResultAsc.getResults().get(0).getValues().get(5).getDoubleValue());
53+
Assertions.assertEquals("service_B", topNResultAsc.getResults().get(0).getValues().get(5).getId());
54+
topNResults.get(2).setType(ExpressionResultType.RECORD_LIST);
55+
Assertions.assertThrows(IllegalExpressionException.class, () -> {
56+
TopNOfOp.doMergeTopNResult(topNResults, 2, MQEParser.DES);
57+
});
58+
}
59+
}

‎oap-server/server-query-plugin/query-graphql-plugin/src/main/java/org/apache/skywalking/oap/query/graphql/mqe/rt/MQEVisitor.java‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -124,8 +124,8 @@ public ExpressionResult visitMetric(MQEParser.MetricContext ctx) {
124124
Column.ValueDataType dataType = valueColumn.get().getDataType();
125125
try {
126126
if (Column.ValueDataType.COMMON_VALUE == dataType) {
127-
if (ctx.parent instanceof MQEParser.TopNOPContext) {
128-
MQEParser.TopNOPContext parent = (MQEParser.TopNOPContext) ctx.parent;
127+
if (ctx.parent instanceof MQEParser.TopNContext) {
128+
MQEParser.TopNContext parent = (MQEParser.TopNContext) ctx.parent;
129129
int topN = Integer.parseInt(parent.INTEGER().getText());
130130
if (topN <= 0) {
131131
throw new IllegalExpressionException("TopN value must be > 0.");
@@ -170,8 +170,8 @@ public ExpressionResult visitMetric(MQEParser.MetricContext ctx) {
170170
queryLabeledMetrics(metricName, queryLabels, this.duration, result);
171171
}
172172
} else if (Column.ValueDataType.SAMPLED_RECORD == dataType) {
173-
if (ctx.parent instanceof MQEParser.TopNOPContext) {
174-
MQEParser.TopNOPContext parent = (MQEParser.TopNOPContext) ctx.parent;
173+
if (ctx.parent instanceof MQEParser.TopNContext) {
174+
MQEParser.TopNContext parent = (MQEParser.TopNContext) ctx.parent;
175175
int topN = Integer.parseInt(parent.INTEGER().getText());
176176
if (topN <= 0) {
177177
throw new IllegalExpressionException("TopN value must be > 0.");

‎test/e2e-v2/cases/mqe/mqe-cases.yaml‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,10 +45,13 @@ cases:
4545
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_resp_time,3,des,attr0!='Not_GENERAL')/100"
4646
expected: expected/topN-OP-service.yml
4747

48-
# topN-OP-isntance
48+
# topN-OP-instance
4949
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n(service_instance_resp_time,3,des)/100" --service-name=e2e-service-provider
5050
expected: expected/topN-OP-instance.yml
5151

52+
# topN-Of-OP
53+
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="top_n_of(top_n(service_resp_time,3,des,attr0='GENERAL'), top_n(service_cpm,3,des,attr0='GENERAL'), 2, des)"
54+
expected: expected/topN-OP-service.yml
5255
# select labels and relabels
5356
- query: swctl --display yaml --base-url=http://${oap_host}:${oap_12800}/graphql metrics exec --expression="relabels(relabels(service_percentile{p='50,75,90'},p='50,75',p='P50,P75'),p='90',p='P90')" --service-name=e2e-service-provider
5457
expected: expected/relabels-OP.yml

0 commit comments

Comments
 (0)