Skip to content

Commit 3441027

Browse files
peachisaipeachisai
authored
Aggregate slow statement records by service dimension (#13514)
1 parent c48cdf3 commit 3441027

File tree

8 files changed

+268
-10
lines changed

8 files changed

+268
-10
lines changed

‎docs/en/changes/changes.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -101,6 +101,7 @@
101101
* Bump up netty to 4.2.5.Final.
102102
* BanyanDB: fix log query missing order by condition, and fix missing service id condition when query by instance id or endpoint id.
103103
* Fix potential NPE in the `AlarmStatusQueryHandler`.
104+
* Aggregate TopN Slow SQL by service dimension.
104105

105106
#### UI
106107

‎oap-server/analyzer/agent-analyzer/src/main/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessor.java‎

Lines changed: 28 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,7 @@
3333
import org.apache.skywalking.oap.server.core.config.NamingControl;
3434
import org.apache.skywalking.oap.server.core.source.DatabaseAccess;
3535
import org.apache.skywalking.oap.server.core.source.DatabaseSlowStatement;
36+
import org.apache.skywalking.oap.server.core.source.ServiceDatabaseSlowStatement;
3637
import org.apache.skywalking.oap.server.core.source.ServiceMeta;
3738
import org.apache.skywalking.oap.server.core.source.Source;
3839
import org.apache.skywalking.oap.server.library.util.StringUtil;
@@ -65,18 +66,36 @@ public void prepareVSIfNecessary(SpanObject span, SegmentObject segmentObject) {
6566
recordList.add(toDatabaseAccess(span, serviceName, timeBucket, latency));
6667

6768
readStatementIfSlow(span.getTagsList(), latency).ifPresent(statement -> {
68-
DatabaseSlowStatement dbSlowStat = new DatabaseSlowStatement();
69-
dbSlowStat.setId(segmentObject.getTraceSegmentId() + "-" + span.getSpanId());
70-
dbSlowStat.setTraceId(segmentObject.getTraceId());
71-
dbSlowStat.setDatabaseServiceId(IDManager.ServiceID.buildId(serviceName, false));
72-
dbSlowStat.setStatement(statement);
73-
dbSlowStat.setLatency(latency);
74-
dbSlowStat.setTimeBucket(TimeBucket.getRecordTimeBucket(span.getStartTime()));
75-
dbSlowStat.setTimestamp(span.getStartTime());
76-
recordList.add(dbSlowStat);
69+
recordList.add(buildDatabaseSlowStatement(span, segmentObject, statement, serviceName, latency));
70+
recordList.add(buildServiceDatabaseSlowStatement(span, segmentObject, statement, latency));
7771
});
7872
}
7973

74+
private DatabaseSlowStatement buildDatabaseSlowStatement(SpanObject span, SegmentObject segmentObject, String statement, String serviceName, int latency) {
75+
DatabaseSlowStatement dbSlowStat = new DatabaseSlowStatement();
76+
dbSlowStat.setId(segmentObject.getTraceSegmentId() + "-" + span.getSpanId());
77+
dbSlowStat.setTraceId(segmentObject.getTraceId());
78+
dbSlowStat.setDatabaseServiceId(IDManager.ServiceID.buildId(serviceName, false));
79+
dbSlowStat.setStatement(statement);
80+
dbSlowStat.setLatency(latency);
81+
dbSlowStat.setTimeBucket(TimeBucket.getRecordTimeBucket(span.getStartTime()));
82+
dbSlowStat.setTimestamp(span.getStartTime());
83+
return dbSlowStat;
84+
}
85+
86+
private ServiceDatabaseSlowStatement buildServiceDatabaseSlowStatement(SpanObject span, SegmentObject segmentObject, String statement, int latency) {
87+
ServiceDatabaseSlowStatement serviceDbSlowStat = new ServiceDatabaseSlowStatement();
88+
serviceDbSlowStat.setId(segmentObject.getTraceSegmentId() + "-" + span.getSpanId());
89+
serviceDbSlowStat.setTraceId(segmentObject.getTraceId());
90+
serviceDbSlowStat.setServiceId(IDManager.ServiceID.buildId(segmentObject.getService(), true));
91+
serviceDbSlowStat.setStatement(statement);
92+
serviceDbSlowStat.setLatency(latency);
93+
serviceDbSlowStat.setTimeBucket(TimeBucket.getRecordTimeBucket(span.getStartTime()));
94+
serviceDbSlowStat.setTimestamp(span.getStartTime());
95+
96+
return serviceDbSlowStat;
97+
}
98+
8099
private Optional<String> readStatementIfSlow(List<KeyStringValuePair> tags, int latency) {
81100
String statement = null;
82101
boolean isSlowDBAccess = false;

‎oap-server/analyzer/agent-analyzer/src/test/java/org/apache/skywalking/oap/server/analyzer/provider/trace/parser/listener/vservice/VirtualDatabaseProcessorTest.java‎

Lines changed: 10 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,7 @@
3232
import org.apache.skywalking.oap.server.core.config.group.EndpointNameGrouping;
3333
import org.apache.skywalking.oap.server.core.source.DatabaseAccess;
3434
import org.apache.skywalking.oap.server.core.source.DatabaseSlowStatement;
35+
import org.apache.skywalking.oap.server.core.source.ServiceDatabaseSlowStatement;
3536
import org.apache.skywalking.oap.server.core.source.ServiceMeta;
3637
import org.apache.skywalking.oap.server.core.source.Source;
3738
import org.joda.time.DateTime;
@@ -70,12 +71,13 @@ public void testExitSpan() {
7071
.build();
7172
SegmentObject segmentObject = SegmentObject.newBuilder()
7273
.setTraceId("trace-id-1")
74+
.setService("test-service")
7375
.build();
7476
VirtualDatabaseProcessor processor = buildVirtualServiceProcessor();
7577
processor.prepareVSIfNecessary(spanObject, segmentObject);
7678
ArrayList<Source> sources = new ArrayList<>();
7779
processor.emitTo(sources::add);
78-
Assertions.assertEquals(sources.size(), 3);
80+
Assertions.assertEquals(sources.size(), 4);
7981

8082
ServiceMeta serviceMeta = (ServiceMeta) sources.get(0);
8183
Assertions.assertEquals("127.0.0.1:3306", serviceMeta.getName());
@@ -92,6 +94,13 @@ public void testExitSpan() {
9294
Assertions.assertEquals(1000, slowStatement.getLatency());
9395
Assertions.assertEquals(20220912141312L, slowStatement.getTimeBucket());
9496
Assertions.assertEquals("trace-id-1", slowStatement.getTraceId());
97+
98+
ServiceDatabaseSlowStatement serviceDatabaseSlowStatement = (ServiceDatabaseSlowStatement) sources.get(3);
99+
Assertions.assertEquals("dGVzdC1zZXJ2aWNl.1", serviceDatabaseSlowStatement.getServiceId());
100+
Assertions.assertEquals(1000, slowStatement.getLatency());
101+
Assertions.assertEquals(20220912141312L, slowStatement.getTimeBucket());
102+
Assertions.assertEquals("trace-id-1", slowStatement.getTraceId());
103+
95104
}
96105

97106
@Test

‎oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/database/ServiceDatabaseSlowStatementDispatcher.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,39 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package org.apache.skywalking.oap.server.core.analysis.manual.database;
20+
21+
import org.apache.skywalking.oap.server.core.analysis.SourceDispatcher;
22+
import org.apache.skywalking.oap.server.core.analysis.worker.TopNStreamProcessor;
23+
import org.apache.skywalking.oap.server.core.source.ServiceDatabaseSlowStatement;
24+
25+
public class ServiceDatabaseSlowStatementDispatcher implements SourceDispatcher<ServiceDatabaseSlowStatement> {
26+
@Override
27+
public void dispatch(ServiceDatabaseSlowStatement source) {
28+
TopNServiceDatabaseStatement statement = new TopNServiceDatabaseStatement();
29+
statement.setId(source.getId());
30+
statement.setEntityId(source.getServiceId());
31+
statement.setLatency(source.getLatency());
32+
statement.setStatement(source.getStatement());
33+
statement.setTimeBucket(source.getTimeBucket());
34+
statement.setTraceId(source.getTraceId());
35+
statement.setTimestamp(source.getTimestamp());
36+
37+
TopNStreamProcessor.getInstance().in(statement);
38+
}
39+
}

‎oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/analysis/manual/database/TopNServiceDatabaseStatement.java‎

Lines changed: 96 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,96 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package org.apache.skywalking.oap.server.core.analysis.manual.database;
20+
21+
import lombok.Getter;
22+
import lombok.Setter;
23+
import org.apache.skywalking.oap.server.core.analysis.Stream;
24+
import org.apache.skywalking.oap.server.core.analysis.topn.TopN;
25+
import org.apache.skywalking.oap.server.core.analysis.worker.TopNStreamProcessor;
26+
import org.apache.skywalking.oap.server.core.source.DefaultScopeDefine;
27+
import org.apache.skywalking.oap.server.core.storage.StorageID;
28+
import org.apache.skywalking.oap.server.core.storage.annotation.BanyanDB;
29+
import org.apache.skywalking.oap.server.core.storage.annotation.Column;
30+
import org.apache.skywalking.oap.server.core.storage.type.Convert2Entity;
31+
import org.apache.skywalking.oap.server.core.storage.type.Convert2Storage;
32+
import org.apache.skywalking.oap.server.core.storage.type.StorageBuilder;
33+
34+
import java.util.Objects;
35+
36+
/**
37+
* Service Database TopN statement, including Database SQL statement, mongoDB and Redis commands.
38+
*/
39+
@Stream(name = TopNServiceDatabaseStatement.INDEX_NAME, scopeId = DefaultScopeDefine.SERVICE_DATABASE_SLOW_STATEMENT, builder = TopNServiceDatabaseStatement.Builder.class, processor = TopNStreamProcessor.class)
40+
@BanyanDB.TimestampColumn(TopN.TIMESTAMP)
41+
@BanyanDB.Group(streamGroup = BanyanDB.StreamGroup.RECORDS)
42+
public class TopNServiceDatabaseStatement extends TopN {
43+
44+
public static final String INDEX_NAME = "top_n_service_database_statement";
45+
46+
@Setter
47+
private String id;
48+
@Getter
49+
@Setter
50+
@Column(name = STATEMENT, length = 2000, storageOnly = true)
51+
private String statement;
52+
53+
@Override
54+
public StorageID id() {
55+
return new StorageID().append(id);
56+
}
57+
58+
@Override
59+
public boolean equals(Object o) {
60+
if (this == o)
61+
return true;
62+
if (o == null || getClass() != o.getClass())
63+
return false;
64+
TopNServiceDatabaseStatement statement = (TopNServiceDatabaseStatement) o;
65+
return Objects.equals(getEntityId(), statement.getEntityId());
66+
}
67+
68+
@Override
69+
public int hashCode() {
70+
return Objects.hash(getEntityId());
71+
}
72+
73+
public static class Builder implements StorageBuilder<TopNServiceDatabaseStatement> {
74+
@Override
75+
public TopNServiceDatabaseStatement storage2Entity(final Convert2Entity converter) {
76+
TopNServiceDatabaseStatement statement = new TopNServiceDatabaseStatement();
77+
statement.setStatement((String) converter.get(STATEMENT));
78+
statement.setTraceId((String) converter.get(TRACE_ID));
79+
statement.setLatency(((Number) converter.get(LATENCY)).longValue());
80+
statement.setEntityId((String) converter.get(ENTITY_ID));
81+
statement.setTimeBucket(((Number) converter.get(TIME_BUCKET)).longValue());
82+
statement.setTimestamp(((Number) converter.get(TIMESTAMP)).longValue());
83+
return statement;
84+
}
85+
86+
@Override
87+
public void entity2Storage(final TopNServiceDatabaseStatement storageData, final Convert2Storage converter) {
88+
converter.accept(STATEMENT, storageData.getStatement());
89+
converter.accept(TRACE_ID, storageData.getTraceId());
90+
converter.accept(LATENCY, storageData.getLatency());
91+
converter.accept(ENTITY_ID, storageData.getEntityId());
92+
converter.accept(TIME_BUCKET, storageData.getTimeBucket());
93+
converter.accept(TIMESTAMP, storageData.getTimestamp());
94+
}
95+
}
96+
}

‎oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/DefaultScopeDefine.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -151,6 +151,7 @@ public class DefaultScopeDefine {
151151
public static final int BROWSER_APP_RESOURCE_PERF = 88;
152152
public static final int BROWSER_APP_WEB_INTERACTION_PAGE_PERF = 89;
153153
public static final int SW_SPAN_ATTACHED_EVENT = 90;
154+
public static final int SERVICE_DATABASE_SLOW_STATEMENT = 91;
154155

155156
/**
156157
* Catalog of scope, the metrics processor could use this to group all generated metrics by oal rt.

‎oap-server/server-core/src/main/java/org/apache/skywalking/oap/server/core/source/ServiceDatabaseSlowStatement.java‎

Lines changed: 67 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,67 @@
1+
/*
2+
* Licensed to the Apache Software Foundation (ASF) under one or more
3+
* contributor license agreements. See the NOTICE file distributed with
4+
* this work for additional information regarding copyright ownership.
5+
* The ASF licenses this file to You under the Apache License, Version 2.0
6+
* (the "License"); you may not use this file except in compliance with
7+
* the License. You may obtain a copy of the License at
8+
*
9+
* http://www.apache.org/licenses/LICENSE-2.0
10+
*
11+
* Unless required by applicable law or agreed to in writing, software
12+
* distributed under the License is distributed on an "AS IS" BASIS,
13+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14+
* See the License for the specific language governing permissions and
15+
* limitations under the License.
16+
*
17+
*/
18+
19+
package org.apache.skywalking.oap.server.core.source;
20+
21+
import lombok.Getter;
22+
import lombok.Setter;
23+
import org.apache.skywalking.oap.server.core.Const;
24+
25+
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_CATALOG_NAME;
26+
import static org.apache.skywalking.oap.server.core.source.DefaultScopeDefine.SERVICE_DATABASE_SLOW_STATEMENT;
27+
28+
@ScopeDeclaration(id = SERVICE_DATABASE_SLOW_STATEMENT, name = "ServiceDatabaseSlowStatement", catalog = SERVICE_CATALOG_NAME)
29+
public class ServiceDatabaseSlowStatement extends Source {
30+
31+
@Getter
32+
@Setter
33+
private String id;
34+
35+
@Getter
36+
@Setter
37+
@ScopeDefaultColumn.DefinedByField(columnName = "service_id")
38+
@ScopeDefaultColumn.BanyanDB(shardingKeyIdx = 0)
39+
private String serviceId;
40+
41+
@Getter
42+
@Setter
43+
private String statement;
44+
45+
@Getter
46+
@Setter
47+
private long latency;
48+
49+
@Getter
50+
@Setter
51+
private String traceId;
52+
53+
@Getter
54+
@Setter
55+
private long timestamp;
56+
57+
@Override
58+
public int scope() {
59+
return DefaultScopeDefine.SERVICE_DATABASE_SLOW_STATEMENT;
60+
}
61+
62+
@Override
63+
public String getEntityId() {
64+
return Const.EMPTY_STRING;
65+
}
66+
67+
}

‎oap-server/server-starter/src/main/resources/ui-initialized-templates/general/general-service.json‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -860,6 +860,32 @@
860860
"type": "Log"
861861
}
862862
]
863+
},
864+
{
865+
"name": "Slow Statements",
866+
"children": [
867+
{
868+
"x": 0,
869+
"y": 0,
870+
"w": 24,
871+
"h": 48,
872+
"i": "0",
873+
"type": "Widget",
874+
"widget": {
875+
"title": "Slow Statements (ms)"
876+
},
877+
"graph": {
878+
"type": "TopList",
879+
"color": "purple"
880+
},
881+
"expressions": [
882+
"top_n(top_n_service_database_statement,20,des)"
883+
],
884+
"relatedTrace": {
885+
"refIdType": "traceId"
886+
}
887+
}
888+
]
863889
}
864890
]
865891
},

0 commit comments

Comments
 (0)