Skip to content

Commit bdd0c92

Browse files
hanahmilyhanahmily
authored
Add batch settings and fix trace query bug (#12749)
Signed-off-by: Gao Hongtao <[email protected]>
1 parent 99162ef commit bdd0c92

File tree

14 files changed

+90
-17
lines changed

14 files changed

+90
-17
lines changed

‎docs/en/changes/changes.md‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,8 @@
1616
* Add metrics value owner for metrics topN query result.
1717
* Add naming control for `EndpointDependencyBuilder`.
1818
* The index type `BanyanDB.IndexRule.IndexType#TREE` is removed. All indices are using `IndexType#INVERTED` now.
19+
* Add max query size settings to BanyanDB.
20+
* Fix "BanyanDBTraceQueryDAO.queryBasicTraces" doesn't support querying by "trace_id".
1921

2022
#### UI
2123

‎docs/en/setup/backend/configuration-vocabulary.md‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,11 @@ The Configuration Vocabulary lists all available configurations provided by `app
150150
| - | - | superDatasetShardsFactor | Shards Factor for a super dataset record, i.e. Shard number of a super dataset is recordShardsNumber*superDatasetShardsFactor. | SW_STORAGE_BANYANDB_SUPERDATASET_SHARDS_FACTOR | 2 |
151151
| - | - | concurrentWriteThreads | Concurrent consumer threads for batch writing. | SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS | 15 |
152152
| - | - | profileTaskQueryMaxSize | Max size of ProfileTask to be fetched. | SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE | 200 |
153+
| - | - | resultWindowMaxSize | The maximum size of dataset when the OAP loads cache, such as network aliases. | SW_STORAGE_BANYAND_QUERY_MAX_WINDOW_SIZE | 10000 |
154+
| - | - | metadataQueryMaxSize | The maximum size of metadata per query. | SW_STORAGE_BANYAND_QUERY_MAX_SIZE | 10000 |
155+
| - | - | segmentQueryMaxSize | The maximum size of trace segments per query. | SW_STORAGE_BANYAND_QUERY_SEGMENT_SIZE | 200 |
153156
| - | - | asyncProfilerTaskQueryMaxSize | Max size of AsyncProfilerTask to be fetched. | SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE| 200 |
157+
| - | - | profileDataQueryScrollBatchSize | The batch size of query profiling data. | SW_STORAGE_BANYAND_QUERY_PROFILE_DATA_BATCH_SIZE | 100 |
154158
| - | - | segmentIntervalDays | Data is stored in BanyanDB in segments. A segment is a time range of data.The segment interval is the time range of a segment.The value should be less or equal to data TTL relevant settings. | SW_STORAGE_BANYANDB_SEGMENT_INTERVAL_DAYS | 1 |
155159
| - | - | superDatasetSegmentIntervalDays | The super dataset segment interval is the time range of a segment in the super dataset. | SW_STORAGE_BANYANDB_SUPER_DATASET_SEGMENT_INTERVAL_DAYS | 1 |
156160
| - | - | specificGroupSettings | Specific groups settings. | SW_STORAGE_BANYANDB_SPECIFIC_GROUP_SETTINGS | - |

‎docs/en/setup/backend/storages/banyandb.md‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -36,8 +36,16 @@ storage:
3636
# The number of threads that write data to BanyanDB concurrently.
3737
# Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server CPU usage.
3838
concurrentWriteThreads: ${SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS:15}
39+
# The maximum size of dataset when the OAP loads cache, such as network aliases.
40+
resultWindowMaxSize: ${SW_STORAGE_BANYANDB_QUERY_MAX_WINDOW_SIZE:10000}
41+
# The maximum size of metadata per query.
42+
metadataQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_MAX_SIZE:10000}
43+
# The maximum size of trace segments per query.
44+
segmentQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_SEGMENT_SIZE:200}
3945
# The max number of profile task query in a request.
40-
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200}
46+
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_TASK_SIZE:200}
47+
# The batch size of query profiling data.
48+
profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100}
4149
# Data is stored in BanyanDB in segments. A segment is a time range of data.
4250
# The segment interval is the time range of a segment.
4351
# The value should be less or equal to data TTL relevant settings.

‎oap-server/server-starter/src/main/resources/application.yml‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -245,8 +245,16 @@ storage:
245245
# The number of threads that write data to BanyanDB concurrently.
246246
# Bigger value can improve the write performance, but also increase the OAP and BanyanDB Server CPU usage.
247247
concurrentWriteThreads: ${SW_STORAGE_BANYANDB_CONCURRENT_WRITE_THREADS:15}
248+
# The maximum size of dataset when the OAP loads cache, such as network aliases.
249+
resultWindowMaxSize: ${SW_STORAGE_BANYANDB_QUERY_MAX_WINDOW_SIZE:10000}
250+
# The maximum size of metadata per query.
251+
metadataQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_MAX_SIZE:10000}
252+
# The maximum size of trace segments per query.
253+
segmentQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_SEGMENT_SIZE:200}
248254
# The max number of profile task query in a request.
249-
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_PROFILE_TASK_QUERY_MAX_SIZE:200}
255+
profileTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_TASK_SIZE:200}
256+
# The batch size of query profiling data.
257+
profileDataQueryBatchSize: ${SW_STORAGE_BANYANDB_QUERY_PROFILE_DATA_BATCH_SIZE:100}
250258
asyncProfilerTaskQueryMaxSize: ${SW_STORAGE_BANYANDB_ASYNC_PROFILER_TASK_QUERY_MAX_SIZE:200}
251259
# Data is stored in BanyanDB in segments. A segment is a time range of data.
252260
# The segment interval is the time range of a segment.

‎oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageConfig.java‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -103,6 +103,11 @@ public class BanyanDBStorageConfig extends ModuleConfig {
103103
*/
104104
private int asyncProfilerTaskQueryMaxSize;
105105

106+
private int resultWindowMaxSize = 10000;
107+
private int metadataQueryMaxSize = 5000;
108+
private int segmentQueryMaxSize = 200;
109+
private int profileDataQueryBatchSize = 100;
110+
106111
public String[] getTargetArray() {
107112
return Iterables.toArray(Splitter.on(",").omitEmptyStrings().trimResults().split(this.targets), String.class);
108113
}

‎oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/BanyanDBStorageProvider.java‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -132,10 +132,10 @@ IBatchDAO.class, new BanyanDBBatchDAO(client, config.getMaxBulkSize(), config.ge
132132
config.getConcurrentWriteThreads()
133133
));
134134
this.registerServiceImplementation(StorageDAO.class, new BanyanDBStorageDAO(client));
135-
this.registerServiceImplementation(INetworkAddressAliasDAO.class, new BanyanDBNetworkAddressAliasDAO(client));
136-
this.registerServiceImplementation(ITraceQueryDAO.class, new BanyanDBTraceQueryDAO(client));
135+
this.registerServiceImplementation(INetworkAddressAliasDAO.class, new BanyanDBNetworkAddressAliasDAO(client, this.config));
136+
this.registerServiceImplementation(ITraceQueryDAO.class, new BanyanDBTraceQueryDAO(client, this.config.getSegmentQueryMaxSize()));
137137
this.registerServiceImplementation(IBrowserLogQueryDAO.class, new BanyanDBBrowserLogQueryDAO(client));
138-
this.registerServiceImplementation(IMetadataQueryDAO.class, new BanyanDBMetadataQueryDAO(client));
138+
this.registerServiceImplementation(IMetadataQueryDAO.class, new BanyanDBMetadataQueryDAO(client, this.config));
139139
this.registerServiceImplementation(IAlarmQueryDAO.class, new BanyanDBAlarmQueryDAO(client));
140140
this.registerServiceImplementation(ILogQueryDAO.class, new BanyanDBLogQueryDAO(client));
141141
this.registerServiceImplementation(
@@ -155,20 +155,20 @@ IProfileThreadSnapshotQueryDAO.class, new BanyanDBProfileThreadSnapshotQueryDAO(
155155
this.registerServiceImplementation(IEventQueryDAO.class, new BanyanDBEventQueryDAO(client));
156156
this.registerServiceImplementation(ITopologyQueryDAO.class, new BanyanDBTopologyQueryDAO(client));
157157
this.registerServiceImplementation(IEBPFProfilingTaskDAO.class, new BanyanDBEBPFProfilingTaskDAO(client));
158-
this.registerServiceImplementation(IEBPFProfilingDataDAO.class, new BanyanDBEBPFProfilingDataDAO(client));
158+
this.registerServiceImplementation(IEBPFProfilingDataDAO.class, new BanyanDBEBPFProfilingDataDAO(client, this.config.getProfileDataQueryBatchSize()));
159159
this.registerServiceImplementation(
160160
IEBPFProfilingScheduleDAO.class, new BanyanDBEBPFProfilingScheduleQueryDAO(client));
161161
this.registerServiceImplementation(IContinuousProfilingPolicyDAO.class, new BanyanDBContinuousProfilingPolicyDAO(client));
162162

163-
this.registerServiceImplementation(IServiceLabelDAO.class, new BanyanDBServiceLabelDAO(client));
163+
this.registerServiceImplementation(IServiceLabelDAO.class, new BanyanDBServiceLabelDAO(client, this.config));
164164
this.registerServiceImplementation(ITagAutoCompleteQueryDAO.class, new BanyanDBTagAutocompleteQueryDAO(client));
165165
this.registerServiceImplementation(IHistoryDeleteDAO.class, new BanyanDBHistoryDeleteDAO());
166166
this.registerServiceImplementation(IMetricsQueryDAO.class, new BanyanDBMetricsQueryDAO(client));
167167
this.registerServiceImplementation(IAggregationQueryDAO.class, new BanyanDBAggregationQueryDAO(client));
168168
this.registerServiceImplementation(IRecordsQueryDAO.class, new BanyanDBRecordsQueryDAO(client));
169169
this.registerServiceImplementation(IZipkinQueryDAO.class, new BanyanDBZipkinQueryDAO(client));
170-
this.registerServiceImplementation(ISpanAttachedEventQueryDAO.class, new BanyanDBSpanAttachedEventQueryDAO(client));
171-
this.registerServiceImplementation(IHierarchyQueryDAO.class, new BanyanDBHierarchyQueryDAO(client));
170+
this.registerServiceImplementation(ISpanAttachedEventQueryDAO.class, new BanyanDBSpanAttachedEventQueryDAO(client, this.config.getProfileDataQueryBatchSize()));
171+
this.registerServiceImplementation(IHierarchyQueryDAO.class, new BanyanDBHierarchyQueryDAO(client, this.config));
172172
this.registerServiceImplementation(
173173
IAsyncProfilerTaskQueryDAO.class, new BanyanDBAsyncProfilerTaskQueryDAO(client,
174174
this.config.getAsyncProfilerTaskQueryMaxSize()

‎oap-server/server-storage-plugin/storage-banyandb-plugin/src/main/java/org/apache/skywalking/oap/server/storage/plugin/banyandb/measure/BanyanDBHierarchyQueryDAO.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.skywalking.oap.server.core.storage.query.IHierarchyQueryDAO;
3636
import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBConverter;
3737
import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageClient;
38+
import org.apache.skywalking.oap.server.storage.plugin.banyandb.BanyanDBStorageConfig;
3839
import org.apache.skywalking.oap.server.storage.plugin.banyandb.MetadataRegistry;
3940
import org.apache.skywalking.oap.server.storage.plugin.banyandb.stream.AbstractBanyanDBDAO;
4041

@@ -52,9 +53,11 @@ public class BanyanDBHierarchyQueryDAO extends AbstractBanyanDBDAO implements IH
5253
InstanceHierarchyRelationTraffic.RELATED_INSTANCE_ID,
5354
InstanceHierarchyRelationTraffic.RELATED_SERVICE_LAYER
5455
);
56+
private final int limit;
5557

56-
public BanyanDBHierarchyQueryDAO(final BanyanDBStorageClient client) {
58+
public BanyanDBHierarchyQueryDAO(final BanyanDBStorageClient client, BanyanDBStorageConfig config) {
5759
super(client);
60+
this.limit = config.getMetadataQueryMaxSize();
5861
}
5962

6063
@Override
@@ -65,6 +68,7 @@ public List<ServiceHierarchyRelationTraffic> readAllServiceHierarchyRelations()
6568
Collections.emptySet(), new QueryBuilder<>() {
6669
@Override
6770
protected void apply(MeasureQuery query) {
71+
query.limit(limit);
6872
}
6973
}
7074
);

0 commit comments

Comments
 (0)