Skip to content

Commit 666924d

Browse files
biggusdonzusbiggusdonzus
authored
Merge pull request #612 from Aiven-Open/jeqo/docs-metrics-storages
refactor: expose storage backends metric names for doc generation
2 parents a6ec31f + ef05c49 commit 666924d

File tree

6 files changed

+927
-61
lines changed

6 files changed

+927
-61
lines changed

‎storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricCollector.java‎

Lines changed: 55 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -38,22 +38,38 @@
3838
import com.azure.core.http.policy.HttpPipelinePolicy;
3939
import reactor.core.publisher.Mono;
4040

41-
public class MetricCollector {
42-
private final org.apache.kafka.common.metrics.Metrics metrics;
41+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_DELETE;
42+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_DELETE_RATE_METRIC_NAME;
43+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_DELETE_TOTAL_METRIC_NAME;
44+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_GET;
45+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_GET_RATE_METRIC_NAME;
46+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_GET_TOTAL_METRIC_NAME;
47+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_UPLOAD;
48+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_UPLOAD_RATE_METRIC_NAME;
49+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOB_UPLOAD_TOTAL_METRIC_NAME;
50+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_LIST_UPLOAD;
51+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_LIST_UPLOAD_RATE_METRIC_NAME;
52+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_LIST_UPLOAD_TOTAL_METRIC_NAME;
53+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_UPLOAD;
54+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_UPLOAD_RATE_METRIC_NAME;
55+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.BLOCK_UPLOAD_TOTAL_METRIC_NAME;
56+
import static io.aiven.kafka.tieredstorage.storage.azure.MetricRegistry.METRIC_CONTEXT;
4357

44-
private static final String METRIC_GROUP = "azure-blob-storage-client-metrics";
58+
public class MetricCollector {
4559

4660
final AzureBlobStorageConfig config;
61+
final MetricsPolicy policy;
4762

48-
MetricCollector(final AzureBlobStorageConfig config) {
63+
public MetricCollector(final AzureBlobStorageConfig config) {
4964
this.config = config;
5065

5166
final JmxReporter reporter = new JmxReporter();
5267

53-
metrics = new org.apache.kafka.common.metrics.Metrics(
68+
final Metrics metrics = new Metrics(
5469
new MetricConfig(), List.of(reporter), Time.SYSTEM,
55-
new KafkaMetricsContext("aiven.kafka.server.tieredstorage.azure")
70+
new KafkaMetricsContext(METRIC_CONTEXT)
5671
);
72+
policy = new MetricsPolicy(metrics, pathPattern());
5773
}
5874

5975
Pattern pathPattern() {
@@ -64,7 +80,7 @@ Pattern pathPattern() {
6480
}
6581

6682
MetricsPolicy policy() {
67-
return new MetricsPolicy(metrics, pathPattern());
83+
return policy;
6884
}
6985

7086
static class MetricsPolicy implements HttpPipelinePolicy {
@@ -83,17 +99,41 @@ static class MetricsPolicy implements HttpPipelinePolicy {
8399
MetricsPolicy(final Metrics metrics, final Pattern pathPattern) {
84100
this.metrics = metrics;
85101
this.pathPattern = pathPattern;
86-
this.deleteBlobRequests = createSensor("blob-delete");
87-
this.uploadBlobRequests = createSensor("blob-upload");
88-
this.uploadBlockRequests = createSensor("block-upload");
89-
this.uploadBlockListRequests = createSensor("block-list-upload");
90-
this.getBlobRequests = createSensor("blob-get");
102+
this.deleteBlobRequests = createSensor(
103+
BLOB_DELETE,
104+
BLOB_DELETE_RATE_METRIC_NAME,
105+
BLOB_DELETE_TOTAL_METRIC_NAME
106+
);
107+
this.uploadBlobRequests = createSensor(
108+
BLOB_UPLOAD,
109+
BLOB_UPLOAD_RATE_METRIC_NAME,
110+
BLOB_UPLOAD_TOTAL_METRIC_NAME
111+
);
112+
this.uploadBlockRequests = createSensor(
113+
BLOCK_UPLOAD,
114+
BLOCK_UPLOAD_RATE_METRIC_NAME,
115+
BLOCK_UPLOAD_TOTAL_METRIC_NAME
116+
);
117+
this.uploadBlockListRequests = createSensor(
118+
BLOCK_LIST_UPLOAD,
119+
BLOCK_LIST_UPLOAD_RATE_METRIC_NAME,
120+
BLOCK_LIST_UPLOAD_TOTAL_METRIC_NAME
121+
);
122+
this.getBlobRequests = createSensor(
123+
BLOB_GET,
124+
BLOB_GET_RATE_METRIC_NAME,
125+
BLOB_GET_TOTAL_METRIC_NAME
126+
);
91127
}
92128

93-
private Sensor createSensor(final String name) {
129+
private Sensor createSensor(
130+
final String name,
131+
final MetricNameTemplate rateMetricName,
132+
final MetricNameTemplate totalMetricName
133+
) {
94134
return new SensorProvider(metrics, name)
95-
.with(new MetricNameTemplate(name + "-rate", METRIC_GROUP, ""), new Rate())
96-
.with(new MetricNameTemplate(name + "-total", METRIC_GROUP, ""), new CumulativeCount())
135+
.with(rateMetricName, new Rate())
136+
.with(totalMetricName, new CumulativeCount())
97137
.get();
98138
}
99139

‎storage/azure/src/main/java/io/aiven/kafka/tieredstorage/storage/azure/MetricRegistry.java‎

Lines changed: 116 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,116 @@
1+
/*
2+
* Copyright 2024 Aiven Oy
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
package io.aiven.kafka.tieredstorage.storage.azure;
18+
19+
import java.util.List;
20+
21+
import org.apache.kafka.common.MetricNameTemplate;
22+
23+
public class MetricRegistry {
24+
public static final String METRIC_CONTEXT = "aiven.kafka.server.tieredstorage.azure";
25+
26+
static final String METRIC_GROUP = "azure-blob-storage-client-metrics";
27+
static final String BLOB_DELETE = "blob-delete";
28+
static final String BLOB_DELETE_DOC = "object delete operations";
29+
static final String BLOB_DELETE_RATE = BLOB_DELETE + "-rate";
30+
static final String BLOB_DELETE_TOTAL = BLOB_DELETE + "-total";
31+
static final String BLOB_UPLOAD = "blob-upload";
32+
static final String BLOB_UPLOAD_DOC = "object upload operations";
33+
static final String BLOB_UPLOAD_RATE = BLOB_UPLOAD + "-rate";
34+
static final String BLOB_UPLOAD_TOTAL = BLOB_UPLOAD + "-total";
35+
static final String BLOCK_UPLOAD = "block-upload";
36+
static final String BLOCK_UPLOAD_RATE = BLOCK_UPLOAD + "-rate";
37+
static final String BLOCK_UPLOAD_TOTAL = BLOCK_UPLOAD + "-total";
38+
static final String BLOCK_UPLOAD_DOC = "block (blob part) upload operations";
39+
static final String BLOCK_LIST_UPLOAD = "block-list-upload";
40+
static final String BLOCK_LIST_UPLOAD_RATE = BLOCK_LIST_UPLOAD + "-rate";
41+
static final String BLOCK_LIST_UPLOAD_TOTAL = BLOCK_LIST_UPLOAD + "-total";
42+
static final String BLOCK_LIST_UPLOAD_DOC = "block list (making a blob) upload operations";
43+
static final String BLOB_GET = "blob-get";
44+
static final String BLOB_GET_RATE = BLOB_GET + "-rate";
45+
static final String BLOB_GET_TOTAL = BLOB_GET + "-total";
46+
static final String BLOB_GET_DOC = "get object operations";
47+
48+
private static final String RATE_DOC_PREFIX = "Rate of ";
49+
private static final String TOTAL_DOC_PREFIX = "Total number of ";
50+
51+
static final MetricNameTemplate BLOB_DELETE_RATE_METRIC_NAME = new MetricNameTemplate(
52+
BLOB_DELETE_RATE,
53+
METRIC_GROUP,
54+
RATE_DOC_PREFIX + BLOB_DELETE_DOC
55+
);
56+
static final MetricNameTemplate BLOB_DELETE_TOTAL_METRIC_NAME = new MetricNameTemplate(
57+
BLOB_DELETE_TOTAL,
58+
METRIC_GROUP,
59+
TOTAL_DOC_PREFIX + BLOB_DELETE_DOC
60+
);
61+
static final MetricNameTemplate BLOB_UPLOAD_RATE_METRIC_NAME = new MetricNameTemplate(
62+
BLOB_UPLOAD_RATE,
63+
METRIC_GROUP,
64+
RATE_DOC_PREFIX + BLOB_UPLOAD_DOC
65+
);
66+
static final MetricNameTemplate BLOB_UPLOAD_TOTAL_METRIC_NAME = new MetricNameTemplate(
67+
BLOB_UPLOAD_TOTAL,
68+
METRIC_GROUP,
69+
TOTAL_DOC_PREFIX + BLOB_UPLOAD_DOC
70+
);
71+
static final MetricNameTemplate BLOCK_UPLOAD_RATE_METRIC_NAME = new MetricNameTemplate(
72+
BLOCK_UPLOAD_RATE,
73+
METRIC_GROUP,
74+
RATE_DOC_PREFIX + BLOCK_UPLOAD_DOC
75+
);
76+
static final MetricNameTemplate BLOCK_UPLOAD_TOTAL_METRIC_NAME = new MetricNameTemplate(
77+
BLOCK_UPLOAD_TOTAL,
78+
METRIC_GROUP,
79+
TOTAL_DOC_PREFIX + BLOCK_UPLOAD_DOC
80+
);
81+
static final MetricNameTemplate BLOCK_LIST_UPLOAD_RATE_METRIC_NAME = new MetricNameTemplate(
82+
BLOCK_LIST_UPLOAD_RATE,
83+
METRIC_GROUP,
84+
RATE_DOC_PREFIX + BLOCK_LIST_UPLOAD_DOC
85+
);
86+
static final MetricNameTemplate BLOCK_LIST_UPLOAD_TOTAL_METRIC_NAME = new MetricNameTemplate(
87+
BLOCK_LIST_UPLOAD_TOTAL,
88+
METRIC_GROUP,
89+
TOTAL_DOC_PREFIX + BLOCK_LIST_UPLOAD_DOC
90+
);
91+
static final MetricNameTemplate BLOB_GET_RATE_METRIC_NAME = new MetricNameTemplate(
92+
BLOB_GET_RATE,
93+
METRIC_GROUP,
94+
RATE_DOC_PREFIX + BLOB_GET_DOC
95+
);
96+
static final MetricNameTemplate BLOB_GET_TOTAL_METRIC_NAME = new MetricNameTemplate(
97+
BLOB_GET_TOTAL,
98+
METRIC_GROUP,
99+
TOTAL_DOC_PREFIX + BLOB_GET_DOC
100+
);
101+
102+
public List<MetricNameTemplate> all() {
103+
return List.of(
104+
BLOB_DELETE_RATE_METRIC_NAME,
105+
BLOB_DELETE_TOTAL_METRIC_NAME,
106+
BLOB_UPLOAD_RATE_METRIC_NAME,
107+
BLOB_UPLOAD_TOTAL_METRIC_NAME,
108+
BLOCK_UPLOAD_RATE_METRIC_NAME,
109+
BLOCK_UPLOAD_TOTAL_METRIC_NAME,
110+
BLOCK_LIST_UPLOAD_RATE_METRIC_NAME,
111+
BLOCK_LIST_UPLOAD_TOTAL_METRIC_NAME,
112+
BLOB_GET_RATE_METRIC_NAME,
113+
BLOB_GET_TOTAL_METRIC_NAME
114+
);
115+
}
116+
}

‎storage/gcs/src/main/java/io/aiven/kafka/tieredstorage/storage/gcs/MetricCollector.java‎

Lines changed: 53 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import java.util.List;
2121
import java.util.regex.Pattern;
2222

23+
import org.apache.kafka.common.MetricNameTemplate;
2324
import org.apache.kafka.common.metrics.JmxReporter;
2425
import org.apache.kafka.common.metrics.KafkaMetricsContext;
2526
import org.apache.kafka.common.metrics.MetricConfig;
@@ -37,7 +38,24 @@
3738
import com.google.cloud.ServiceOptions;
3839
import com.google.cloud.http.HttpTransportOptions;
3940

40-
class MetricCollector {
41+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.METRIC_CONTEXT;
42+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE;
43+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_RATE_METRIC_NAME;
44+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_DELETE_TOTAL_METRIC_NAME;
45+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET;
46+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET_RATE_METRIC_NAME;
47+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_GET_TOTAL_METRIC_NAME;
48+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET;
49+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET_RATE_METRIC_NAME;
50+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.OBJECT_METADATA_GET_TOTAL_METRIC_NAME;
51+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD;
52+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME;
53+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME;
54+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE;
55+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME;
56+
import static io.aiven.kafka.tieredstorage.storage.gcs.MetricRegistry.RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME;
57+
58+
public class MetricCollector {
4159
private final org.apache.kafka.common.metrics.Metrics metrics;
4260

4361
/**
@@ -64,33 +82,55 @@ class MetricCollector {
6482
static final Pattern OBJECT_UPLOAD_PATH_PATTERN =
6583
Pattern.compile("^/upload/storage/v1/b/([^/]+)/o/?$");
6684

67-
private static final String METRIC_GROUP = "gcs-client-metrics";
68-
6985
private final Sensor getObjectMetadataRequests;
7086
private final Sensor deleteObjectRequests;
7187
private final Sensor resumableUploadInitiateRequests;
7288
private final Sensor resumableChunkUploadRequests;
7389
private final Sensor getObjectRequests;
7490

75-
MetricCollector() {
91+
public MetricCollector() {
7692
final JmxReporter reporter = new JmxReporter();
7793

7894
metrics = new org.apache.kafka.common.metrics.Metrics(
7995
new MetricConfig(), List.of(reporter), Time.SYSTEM,
80-
new KafkaMetricsContext("aiven.kafka.server.tieredstorage.gcs")
96+
new KafkaMetricsContext(METRIC_CONTEXT)
8197
);
8298

83-
getObjectMetadataRequests = createSensor("object-metadata-get");
84-
getObjectRequests = createSensor("object-get");
85-
deleteObjectRequests = createSensor("object-delete");
86-
resumableUploadInitiateRequests = createSensor("resumable-upload-initiate");
87-
resumableChunkUploadRequests = createSensor("resumable-chunk-upload");
99+
getObjectMetadataRequests = createSensor(
100+
OBJECT_METADATA_GET,
101+
OBJECT_METADATA_GET_RATE_METRIC_NAME,
102+
OBJECT_METADATA_GET_TOTAL_METRIC_NAME
103+
);
104+
getObjectRequests = createSensor(
105+
OBJECT_GET,
106+
OBJECT_GET_RATE_METRIC_NAME,
107+
OBJECT_GET_TOTAL_METRIC_NAME
108+
);
109+
deleteObjectRequests = createSensor(
110+
OBJECT_DELETE,
111+
OBJECT_DELETE_RATE_METRIC_NAME,
112+
OBJECT_DELETE_TOTAL_METRIC_NAME
113+
);
114+
resumableUploadInitiateRequests = createSensor(
115+
RESUMABLE_UPLOAD_INITIATE,
116+
RESUMABLE_UPLOAD_INITIATE_RATE_METRIC_NAME,
117+
RESUMABLE_UPLOAD_INITIATE_TOTAL_METRIC_NAME
118+
);
119+
resumableChunkUploadRequests = createSensor(
120+
RESUMABLE_CHUNK_UPLOAD,
121+
RESUMABLE_CHUNK_UPLOAD_RATE_METRIC_NAME,
122+
RESUMABLE_CHUNK_UPLOAD_TOTAL_METRIC_NAME
123+
);
88124
}
89125

90-
private Sensor createSensor(final String name) {
126+
private Sensor createSensor(
127+
final String name,
128+
final MetricNameTemplate rateMetricName,
129+
final MetricNameTemplate totalMetricName
130+
) {
91131
final Sensor sensor = metrics.sensor(name);
92-
sensor.add(metrics.metricName(name + "-rate", METRIC_GROUP), new Rate());
93-
sensor.add(metrics.metricName(name + "-total", METRIC_GROUP), new CumulativeCount());
132+
sensor.add(metrics.metricInstance(rateMetricName), new Rate());
133+
sensor.add(metrics.metricInstance(totalMetricName), new CumulativeCount());
94134
return sensor;
95135
}
96136

0 commit comments

Comments
 (0)