Skip to content

Commit ef05c49

Browse files
jeqojeqo
committed
refactor: expose s3 metric names for doc generation
1 parent 34bac1b commit ef05c49

File tree

2 files changed

+587
-33
lines changed

2 files changed

+587
-33
lines changed

‎storage/s3/src/main/java/io/aiven/kafka/tieredstorage/storage/s3/MetricCollector.java‎

Lines changed: 210 additions & 33 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import java.util.Map;
2323
import java.util.stream.Collectors;
2424

25+
import org.apache.kafka.common.MetricNameTemplate;
2526
import org.apache.kafka.common.metrics.JmxReporter;
2627
import org.apache.kafka.common.metrics.KafkaMetricsContext;
2728
import org.apache.kafka.common.metrics.MetricConfig;
@@ -39,64 +40,240 @@
3940
import software.amazon.awssdk.metrics.MetricCollection;
4041
import software.amazon.awssdk.metrics.MetricPublisher;
4142

43+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_REQUESTS;
44+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME;
45+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME;
46+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_TIME;
47+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME;
48+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.ABORT_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME;
49+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_REQUESTS;
50+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME;
51+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME;
52+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_TIME;
53+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME;
54+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.COMPLETE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME;
55+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CONFIGURED_TIMEOUT_ERRORS;
56+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CONFIGURED_TIMEOUT_ERRORS_RATE_METRIC_NAME;
57+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CONFIGURED_TIMEOUT_ERRORS_TOTAL_METRIC_NAME;
58+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_REQUESTS;
59+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME;
60+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME;
61+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_TIME;
62+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME;
63+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.CREATE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME;
64+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_REQUESTS;
65+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_REQUESTS_RATE_METRIC_NAME;
66+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_REQUESTS_TOTAL_METRIC_NAME;
67+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_TIME;
68+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_TIME_AVG_METRIC_NAME;
69+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECTS_TIME_MAX_METRIC_NAME;
70+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_REQUESTS;
71+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_REQUESTS_RATE_METRIC_NAME;
72+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_REQUESTS_TOTAL_METRIC_NAME;
73+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_TIME;
74+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_TIME_AVG_METRIC_NAME;
75+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.DELETE_OBJECT_TIME_MAX_METRIC_NAME;
76+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_REQUESTS;
77+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_REQUESTS_RATE_METRIC_NAME;
78+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_REQUESTS_TOTAL_METRIC_NAME;
79+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_TIME;
80+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_TIME_AVG_METRIC_NAME;
81+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.GET_OBJECT_TIME_MAX_METRIC_NAME;
82+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.IO_ERRORS;
83+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.IO_ERRORS_RATE_METRIC_NAME;
84+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.IO_ERRORS_TOTAL_METRIC_NAME;
85+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.METRIC_CONTEXT;
86+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.OTHER_ERRORS;
87+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.OTHER_ERRORS_RATE_METRIC_NAME;
88+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.OTHER_ERRORS_TOTAL_METRIC_NAME;
89+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_REQUESTS;
90+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_REQUESTS_RATE_METRIC_NAME;
91+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_REQUESTS_TOTAL_METRIC_NAME;
92+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_TIME;
93+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_TIME_AVG_METRIC_NAME;
94+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.PUT_OBJECT_TIME_MAX_METRIC_NAME;
95+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.SERVER_ERRORS;
96+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.SERVER_ERRORS_RATE_METRIC_NAME;
97+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.SERVER_ERRORS_TOTAL_METRIC_NAME;
98+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.THROTTLING_ERRORS;
99+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.THROTTLING_ERRORS_RATE_METRIC_NAME;
100+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.THROTTLING_ERRORS_TOTAL_METRIC_NAME;
101+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_REQUESTS;
102+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_REQUESTS_RATE_METRIC_NAME;
103+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_REQUESTS_TOTAL_METRIC_NAME;
104+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_TIME;
105+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_TIME_AVG_METRIC_NAME;
106+
import static io.aiven.kafka.tieredstorage.storage.s3.MetricRegistry.UPLOAD_PART_TIME_MAX_METRIC_NAME;
42107
import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.CONFIGURED_TIMEOUT;
43108
import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.IO;
44109
import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.OTHER;
45110
import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.SERVER_ERROR;
46111
import static software.amazon.awssdk.core.internal.metrics.SdkErrorType.THROTTLING;
47112

48-
class MetricCollector implements MetricPublisher {
113+
public class MetricCollector implements MetricPublisher {
49114
private static final Logger log = LoggerFactory.getLogger(MetricCollector.class);
50115

51116
private final org.apache.kafka.common.metrics.Metrics metrics;
52117

53-
private static final String METRIC_GROUP = "s3-client-metrics";
54118
private final Map<String, Sensor> requestMetrics = new HashMap<>();
55119
private final Map<String, Sensor> latencyMetrics = new HashMap<>();
56120
private final Map<String, Sensor> errorMetrics = new HashMap<>();
57121

58-
MetricCollector() {
122+
public MetricCollector() {
59123
final MetricsReporter reporter = new JmxReporter();
60124

61125
metrics = new org.apache.kafka.common.metrics.Metrics(
62126
new MetricConfig(), List.of(reporter), Time.SYSTEM,
63-
new KafkaMetricsContext("aiven.kafka.server.tieredstorage.s3")
64-
);
65-
requestMetrics.put("GetObject", createRequestsSensor("get-object-requests"));
66-
latencyMetrics.put("GetObject", createLatencySensor("get-object-time"));
67-
requestMetrics.put("UploadPart", createRequestsSensor("upload-part-requests"));
68-
latencyMetrics.put("UploadPart", createLatencySensor("upload-part-time"));
69-
requestMetrics.put("CreateMultipartUpload", createRequestsSensor("create-multipart-upload-requests"));
70-
latencyMetrics.put("CreateMultipartUpload", createLatencySensor("create-multipart-upload-time"));
71-
requestMetrics.put("CompleteMultipartUpload", createRequestsSensor("complete-multipart-upload-requests"));
72-
latencyMetrics.put("CompleteMultipartUpload", createLatencySensor("complete-multipart-upload-time"));
73-
requestMetrics.put("PutObject", createRequestsSensor("put-object-requests"));
74-
latencyMetrics.put("PutObject", createLatencySensor("put-object-time"));
75-
requestMetrics.put("DeleteObject", createRequestsSensor("delete-object-requests"));
76-
latencyMetrics.put("DeleteObject", createLatencySensor("delete-object-time"));
77-
requestMetrics.put("DeleteObjects", createRequestsSensor("delete-objects-requests"));
78-
latencyMetrics.put("DeleteObjects", createLatencySensor("delete-objects-time"));
79-
requestMetrics.put("AbortMultipartUpload", createRequestsSensor("abort-multipart-upload-requests"));
80-
latencyMetrics.put("AbortMultipartUpload", createLatencySensor("abort-multipart-upload-time"));
81-
82-
errorMetrics.put(THROTTLING.toString(), createRequestsSensor("throttling-errors"));
83-
errorMetrics.put(SERVER_ERROR.toString(), createRequestsSensor("server-errors"));
84-
errorMetrics.put(CONFIGURED_TIMEOUT.toString(), createRequestsSensor("configured-timeout-errors"));
85-
errorMetrics.put(IO.toString(), createRequestsSensor("io-errors"));
86-
errorMetrics.put(OTHER.toString(), createRequestsSensor("other-errors"));
127+
new KafkaMetricsContext(METRIC_CONTEXT)
128+
);
129+
final Sensor getObjectRequestsSensor = createRequestsSensor(
130+
GET_OBJECT_REQUESTS,
131+
GET_OBJECT_REQUESTS_RATE_METRIC_NAME,
132+
GET_OBJECT_REQUESTS_TOTAL_METRIC_NAME
133+
);
134+
requestMetrics.put("GetObject", getObjectRequestsSensor);
135+
final Sensor getObjectTimeSensor = createLatencySensor(
136+
GET_OBJECT_TIME,
137+
GET_OBJECT_TIME_AVG_METRIC_NAME,
138+
GET_OBJECT_TIME_MAX_METRIC_NAME
139+
);
140+
latencyMetrics.put("GetObject", getObjectTimeSensor);
141+
final Sensor uploadPartRequestsSensor = createRequestsSensor(
142+
UPLOAD_PART_REQUESTS,
143+
UPLOAD_PART_REQUESTS_RATE_METRIC_NAME,
144+
UPLOAD_PART_REQUESTS_TOTAL_METRIC_NAME
145+
);
146+
requestMetrics.put("UploadPart", uploadPartRequestsSensor);
147+
final Sensor uploadPartTimeSensor = createLatencySensor(
148+
UPLOAD_PART_TIME,
149+
UPLOAD_PART_TIME_AVG_METRIC_NAME,
150+
UPLOAD_PART_TIME_MAX_METRIC_NAME
151+
);
152+
latencyMetrics.put("UploadPart", uploadPartTimeSensor);
153+
final Sensor createMpuRequestsSensor = createRequestsSensor(
154+
CREATE_MULTIPART_UPLOAD_REQUESTS,
155+
CREATE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME,
156+
CREATE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME
157+
);
158+
requestMetrics.put("CreateMultipartUpload", createMpuRequestsSensor);
159+
final Sensor createMpuTimeSensor = createLatencySensor(
160+
CREATE_MULTIPART_UPLOAD_TIME,
161+
CREATE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME,
162+
CREATE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME
163+
);
164+
latencyMetrics.put("CreateMultipartUpload", createMpuTimeSensor);
165+
final Sensor completeMpuRequestsSensor = createRequestsSensor(
166+
COMPLETE_MULTIPART_UPLOAD_REQUESTS,
167+
COMPLETE_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME,
168+
COMPLETE_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME
169+
);
170+
requestMetrics.put("CompleteMultipartUpload", completeMpuRequestsSensor);
171+
final Sensor completeMpuTimeSensor = createLatencySensor(
172+
COMPLETE_MULTIPART_UPLOAD_TIME,
173+
COMPLETE_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME,
174+
COMPLETE_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME
175+
);
176+
latencyMetrics.put("CompleteMultipartUpload", completeMpuTimeSensor);
177+
final Sensor putObjectRequestsSensor = createRequestsSensor(
178+
PUT_OBJECT_REQUESTS,
179+
PUT_OBJECT_REQUESTS_RATE_METRIC_NAME,
180+
PUT_OBJECT_REQUESTS_TOTAL_METRIC_NAME
181+
);
182+
requestMetrics.put("PutObject", putObjectRequestsSensor);
183+
final Sensor putObjectTimeSensor = createLatencySensor(
184+
PUT_OBJECT_TIME,
185+
PUT_OBJECT_TIME_AVG_METRIC_NAME,
186+
PUT_OBJECT_TIME_MAX_METRIC_NAME
187+
);
188+
latencyMetrics.put("PutObject", putObjectTimeSensor);
189+
final Sensor deleteObjectRequestsSensor = createRequestsSensor(
190+
DELETE_OBJECT_REQUESTS,
191+
DELETE_OBJECT_REQUESTS_RATE_METRIC_NAME,
192+
DELETE_OBJECT_REQUESTS_TOTAL_METRIC_NAME
193+
);
194+
requestMetrics.put("DeleteObject", deleteObjectRequestsSensor);
195+
final Sensor deleteObjectTimeSensor = createLatencySensor(
196+
DELETE_OBJECT_TIME,
197+
DELETE_OBJECT_TIME_AVG_METRIC_NAME,
198+
DELETE_OBJECT_TIME_MAX_METRIC_NAME
199+
);
200+
latencyMetrics.put("DeleteObject", deleteObjectTimeSensor);
201+
final Sensor deleteObjectsRequestsSensor = createRequestsSensor(
202+
DELETE_OBJECTS_REQUESTS,
203+
DELETE_OBJECTS_REQUESTS_RATE_METRIC_NAME,
204+
DELETE_OBJECTS_REQUESTS_TOTAL_METRIC_NAME
205+
);
206+
requestMetrics.put("DeleteObjects", deleteObjectsRequestsSensor);
207+
final Sensor deleteObjectsTimeSensor = createLatencySensor(
208+
DELETE_OBJECTS_TIME,
209+
DELETE_OBJECTS_TIME_AVG_METRIC_NAME,
210+
DELETE_OBJECTS_TIME_MAX_METRIC_NAME
211+
);
212+
latencyMetrics.put("DeleteObjects", deleteObjectsTimeSensor);
213+
final Sensor abortMpuRequestsSensor = createRequestsSensor(
214+
ABORT_MULTIPART_UPLOAD_REQUESTS,
215+
ABORT_MULTIPART_UPLOAD_REQUESTS_RATE_METRIC_NAME,
216+
ABORT_MULTIPART_UPLOAD_REQUESTS_TOTAL_METRIC_NAME
217+
);
218+
requestMetrics.put("AbortMultipartUpload", abortMpuRequestsSensor);
219+
final Sensor abortMpuTimeSensor = createLatencySensor(
220+
ABORT_MULTIPART_UPLOAD_TIME,
221+
ABORT_MULTIPART_UPLOAD_TIME_AVG_METRIC_NAME,
222+
ABORT_MULTIPART_UPLOAD_TIME_MAX_METRIC_NAME
223+
);
224+
latencyMetrics.put("AbortMultipartUpload", abortMpuTimeSensor);
225+
226+
final Sensor throttlingErrorsSensor = createRequestsSensor(
227+
THROTTLING_ERRORS,
228+
THROTTLING_ERRORS_RATE_METRIC_NAME,
229+
THROTTLING_ERRORS_TOTAL_METRIC_NAME
230+
);
231+
errorMetrics.put(THROTTLING.toString(), throttlingErrorsSensor);
232+
final Sensor serverErrorsSensor = createRequestsSensor(
233+
SERVER_ERRORS,
234+
SERVER_ERRORS_RATE_METRIC_NAME,
235+
SERVER_ERRORS_TOTAL_METRIC_NAME
236+
);
237+
errorMetrics.put(SERVER_ERROR.toString(), serverErrorsSensor);
238+
final Sensor configuredTimeoutErrorsSensor = createRequestsSensor(
239+
CONFIGURED_TIMEOUT_ERRORS,
240+
CONFIGURED_TIMEOUT_ERRORS_RATE_METRIC_NAME,
241+
CONFIGURED_TIMEOUT_ERRORS_TOTAL_METRIC_NAME
242+
);
243+
errorMetrics.put(CONFIGURED_TIMEOUT.toString(), configuredTimeoutErrorsSensor);
244+
final Sensor ioErrorsSensor = createRequestsSensor(
245+
IO_ERRORS,
246+
IO_ERRORS_RATE_METRIC_NAME,
247+
IO_ERRORS_TOTAL_METRIC_NAME
248+
);
249+
errorMetrics.put(IO.toString(), ioErrorsSensor);
250+
final Sensor otherErrorsSensor = createRequestsSensor(
251+
OTHER_ERRORS,
252+
OTHER_ERRORS_RATE_METRIC_NAME,
253+
OTHER_ERRORS_TOTAL_METRIC_NAME
254+
);
255+
errorMetrics.put(OTHER.toString(), otherErrorsSensor);
87256
}
88257

89-
private Sensor createRequestsSensor(final String name) {
258+
private Sensor createRequestsSensor(
259+
final String name,
260+
final MetricNameTemplate rateMetricName,
261+
final MetricNameTemplate totalMetricName
262+
) {
90263
final Sensor sensor = metrics.sensor(name);
91-
sensor.add(metrics.metricName(name + "-rate", METRIC_GROUP), new Rate());
92-
sensor.add(metrics.metricName(name + "-total", METRIC_GROUP), new CumulativeCount());
264+
sensor.add(metrics.metricInstance(rateMetricName), new Rate());
265+
sensor.add(metrics.metricInstance(totalMetricName), new CumulativeCount());
93266
return sensor;
94267
}
95268

96-
private Sensor createLatencySensor(final String name) {
269+
private Sensor createLatencySensor(
270+
final String name,
271+
final MetricNameTemplate avgMetricName,
272+
final MetricNameTemplate maxMetricName
273+
) {
97274
final Sensor sensor = metrics.sensor(name);
98-
sensor.add(metrics.metricName(name + "-max", METRIC_GROUP), new Max());
99-
sensor.add(metrics.metricName(name + "-avg", METRIC_GROUP), new Avg());
275+
sensor.add(metrics.metricInstance(maxMetricName), new Max());
276+
sensor.add(metrics.metricInstance(avgMetricName), new Avg());
100277
return sensor;
101278
}
102279

0 commit comments

Comments
 (0)