Skip to content

Commit 275290a

Browse files
craig[bot]RaduBerinde
craig[bot]
and
RaduBerinde
committed
Merge #157008
157008: kvserver, storage: add bytes compressed/decompressed metrics r=RaduBerinde a=RaduBerinde The motivation for these metrics is to enable estimating the CPU usage impact of switching to a stronger compression method (which applies only to L5/L6 data/value blocks). Epic: none Release note: None Co-authored-by: Radu Berinde <[email protected]>
2 parents 2d5eaef + aedf482 commit 275290a

File tree

4 files changed

+178
-0
lines changed

4 files changed

+178
-0
lines changed

‎docs/generated/metrics/metrics.yaml‎

Lines changed: 72 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -16126,6 +16126,78 @@ layers:
1612616126
unit: COUNT
1612716127
aggregation: AVG
1612816128
derivative: NON_NEGATIVE_DERIVATIVE
16129+
- name: storage.bytes-compressed.l5.data
16130+
exported_name: storage_bytes_compressed_l5_data
16131+
labeled_name: 'storage.bytes-compressed{level: 5, type: data}'
16132+
description: Total number of logical bytes compressed for L5 data blocks.
16133+
y_axis_label: Bytes
16134+
type: COUNTER
16135+
unit: BYTES
16136+
aggregation: AVG
16137+
derivative: NON_NEGATIVE_DERIVATIVE
16138+
- name: storage.bytes-compressed.l5.values
16139+
exported_name: storage_bytes_compressed_l5_values
16140+
labeled_name: 'storage.bytes-compressed{level: 5, type: values}'
16141+
description: Total number of logical bytes compressed for L5 value blocks.
16142+
y_axis_label: Bytes
16143+
type: COUNTER
16144+
unit: BYTES
16145+
aggregation: AVG
16146+
derivative: NON_NEGATIVE_DERIVATIVE
16147+
- name: storage.bytes-compressed.l6.data
16148+
exported_name: storage_bytes_compressed_l6_data
16149+
labeled_name: 'storage.bytes-compressed{level: 6, type: data}'
16150+
description: Total number of logical bytes compressed for L6 data blocks.
16151+
y_axis_label: Bytes
16152+
type: COUNTER
16153+
unit: BYTES
16154+
aggregation: AVG
16155+
derivative: NON_NEGATIVE_DERIVATIVE
16156+
- name: storage.bytes-compressed.l6.values
16157+
exported_name: storage_bytes_compressed_l6_values
16158+
labeled_name: 'storage.bytes-compressed{level: 6, type: values}'
16159+
description: Total number of logical bytes compressed for L6 value blocks.
16160+
y_axis_label: Bytes
16161+
type: COUNTER
16162+
unit: BYTES
16163+
aggregation: AVG
16164+
derivative: NON_NEGATIVE_DERIVATIVE
16165+
- name: storage.bytes-decompressed.l5.data
16166+
exported_name: storage_bytes_decompressed_l5_data
16167+
labeled_name: 'storage.bytes-decompressed{level: 5, type: data}'
16168+
description: Total number of logical bytes decompressed for L5 data blocks.
16169+
y_axis_label: Bytes
16170+
type: COUNTER
16171+
unit: BYTES
16172+
aggregation: AVG
16173+
derivative: NON_NEGATIVE_DERIVATIVE
16174+
- name: storage.bytes-decompressed.l5.values
16175+
exported_name: storage_bytes_decompressed_l5_values
16176+
labeled_name: 'storage.bytes-decompressed{level: 5, type: values}'
16177+
description: Total number of logical bytes decompressed for L5 value blocks.
16178+
y_axis_label: Bytes
16179+
type: COUNTER
16180+
unit: BYTES
16181+
aggregation: AVG
16182+
derivative: NON_NEGATIVE_DERIVATIVE
16183+
- name: storage.bytes-decompressed.l6.data
16184+
exported_name: storage_bytes_decompressed_l6_data
16185+
labeled_name: 'storage.bytes-decompressed{level: 6, type: data}'
16186+
description: Total number of logical bytes decompressed for L6 data blocks.
16187+
y_axis_label: Bytes
16188+
type: COUNTER
16189+
unit: BYTES
16190+
aggregation: AVG
16191+
derivative: NON_NEGATIVE_DERIVATIVE
16192+
- name: storage.bytes-decompressed.l6.values
16193+
exported_name: storage_bytes_decompressed_l6_values
16194+
labeled_name: 'storage.bytes-decompressed{level: 6, type: values}'
16195+
description: Total number of logical bytes decompressed for L6 value blocks.
16196+
y_axis_label: Bytes
16197+
type: COUNTER
16198+
unit: BYTES
16199+
aggregation: AVG
16200+
derivative: NON_NEGATIVE_DERIVATIVE
1612916201
- name: storage.checkpoints
1613016202
exported_name: storage_checkpoints
1613116203
description: |-

‎pkg/kv/kvserver/metrics.go‎

Lines changed: 97 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1236,6 +1236,70 @@ storage.initial_stats_complete becomes true.
12361236
Measurement: "Ratio",
12371237
Unit: metric.Unit_CONST,
12381238
}
1239+
metaBytesCompressedL5Data = metric.Metadata{
1240+
Name: "storage.bytes-compressed.l5.data",
1241+
Help: "Total number of logical bytes compressed for L5 data blocks.",
1242+
Measurement: "Bytes",
1243+
Unit: metric.Unit_BYTES,
1244+
LabeledName: "storage.bytes-compressed",
1245+
StaticLabels: metric.MakeLabelPairs(metric.LabelLevel, "5", metric.LabelType, "data"),
1246+
}
1247+
metaBytesDecompressedL5Data = metric.Metadata{
1248+
Name: "storage.bytes-decompressed.l5.data",
1249+
Help: "Total number of logical bytes decompressed for L5 data blocks.",
1250+
Measurement: "Bytes",
1251+
Unit: metric.Unit_BYTES,
1252+
LabeledName: "storage.bytes-decompressed",
1253+
StaticLabels: metric.MakeLabelPairs(metric.LabelLevel, "5", metric.LabelType, "data"),
1254+
}
1255+
metaBytesCompressedL5Values = metric.Metadata{
1256+
Name: "storage.bytes-compressed.l5.values",
1257+
Help: "Total number of logical bytes compressed for L5 value blocks.",
1258+
Measurement: "Bytes",
1259+
Unit: metric.Unit_BYTES,
1260+
LabeledName: "storage.bytes-compressed",
1261+
StaticLabels: metric.MakeLabelPairs(metric.LabelLevel, "5", metric.LabelType, "values"),
1262+
}
1263+
metaBytesDecompressedL5Values = metric.Metadata{
1264+
Name: "storage.bytes-decompressed.l5.values",
1265+
Help: "Total number of logical bytes decompressed for L5 value blocks.",
1266+
Measurement: "Bytes",
1267+
Unit: metric.Unit_BYTES,
1268+
LabeledName: "storage.bytes-decompressed",
1269+
StaticLabels: metric.MakeLabelPairs(metric.LabelLevel, "5", metric.LabelType, "values"),
1270+
}
1271+
metaBytesCompressedL6Data = metric.Metadata{
1272+
Name: "storage.bytes-compressed.l6.data",
1273+
Help: "Total number of logical bytes compressed for L6 data blocks.",
1274+
Measurement: "Bytes",
1275+
Unit: metric.Unit_BYTES,
1276+
LabeledName: "storage.bytes-compressed",
1277+
StaticLabels: metric.MakeLabelPairs(metric.LabelLevel, "6", metric.LabelType, "data"),
1278+
}
1279+
metaBytesDecompressedL6Data = metric.Metadata{
1280+
Name: "storage.bytes-decompressed.l6.data",
1281+
Help: "Total number of logical bytes decompressed for L6 data blocks.",
1282+
Measurement: "Bytes",
1283+
Unit: metric.Unit_BYTES,
1284+
LabeledName: "storage.bytes-decompressed",
1285+
StaticLabels: metric.MakeLabelPairs(metric.LabelLevel, "6", metric.LabelType, "data"),
1286+
}
1287+
metaBytesCompressedL6Values = metric.Metadata{
1288+
Name: "storage.bytes-compressed.l6.values",
1289+
Help: "Total number of logical bytes compressed for L6 value blocks.",
1290+
Measurement: "Bytes",
1291+
Unit: metric.Unit_BYTES,
1292+
LabeledName: "storage.bytes-compressed",
1293+
StaticLabels: metric.MakeLabelPairs(metric.LabelLevel, "6", metric.LabelType, "values"),
1294+
}
1295+
metaBytesDecompressedL6Values = metric.Metadata{
1296+
Name: "storage.bytes-decompressed.l6.values",
1297+
Help: "Total number of logical bytes decompressed for L6 value blocks.",
1298+
Measurement: "Bytes",
1299+
Unit: metric.Unit_BYTES,
1300+
LabeledName: "storage.bytes-decompressed",
1301+
StaticLabels: metric.MakeLabelPairs(metric.LabelLevel, "6", metric.LabelType, "values"),
1302+
}
12391303
)
12401304

12411305
var (
@@ -2935,6 +2999,9 @@ Note that the measurement does not include the duration for replicating the eval
29352999
)
29363000

29373001
// StoreMetrics is the set of metrics for a given store.
3002+
//
3003+
// Note: any non-embedded struct fields must implement the metric.Struct
3004+
// interface.
29383005
type StoreMetrics struct {
29393006
registry *metric.Registry
29403007

@@ -3105,6 +3172,7 @@ type StoreMetrics struct {
31053172
SSTableRemoteBytes *metric.Gauge
31063173
SSTableRemoteCount *metric.Gauge
31073174

3175+
// Compression metrics for currently live sstables and blob files.
31083176
CompressionSnappyBytes *metric.Gauge
31093177
CompressionSnappyCR *metric.GaugeFloat64
31103178
CompressionMinLZBytes *metric.Gauge
@@ -3115,6 +3183,16 @@ type StoreMetrics struct {
31153183
CompressionUnknownBytes *metric.Gauge
31163184
CompressionOverallCR *metric.GaugeFloat64
31173185

3186+
// Runtime metrics for compression.
3187+
BytesCompressedL5Values *metric.Counter
3188+
BytesCompressedL5Data *metric.Counter
3189+
BytesCompressedL6Values *metric.Counter
3190+
BytesCompressedL6Data *metric.Counter
3191+
BytesDecompressedL5Values *metric.Counter
3192+
BytesDecompressedL5Data *metric.Counter
3193+
BytesDecompressedL6Values *metric.Counter
3194+
BytesDecompressedL6Data *metric.Counter
3195+
31183196
categoryIterMetrics pebbleCategoryIterMetricsContainer
31193197
categoryDiskWriteMetrics pebbleCategoryDiskWriteMetricsContainer
31203198
ValueSeparationBytesReferenced *metric.Gauge
@@ -3872,6 +3950,15 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
38723950
CompressionUnknownBytes: metric.NewGauge(metaCompressionUnknownBytes),
38733951
CompressionOverallCR: metric.NewGaugeFloat64(metaCompressionOverallCR),
38743952

3953+
BytesCompressedL5Data: metric.NewCounter(metaBytesCompressedL5Data),
3954+
BytesCompressedL5Values: metric.NewCounter(metaBytesCompressedL5Values),
3955+
BytesCompressedL6Data: metric.NewCounter(metaBytesCompressedL6Data),
3956+
BytesCompressedL6Values: metric.NewCounter(metaBytesCompressedL6Values),
3957+
BytesDecompressedL5Data: metric.NewCounter(metaBytesDecompressedL5Data),
3958+
BytesDecompressedL5Values: metric.NewCounter(metaBytesDecompressedL5Values),
3959+
BytesDecompressedL6Data: metric.NewCounter(metaBytesDecompressedL6Data),
3960+
BytesDecompressedL6Values: metric.NewCounter(metaBytesDecompressedL6Values),
3961+
38753962
categoryDiskWriteMetrics: pebbleCategoryDiskWriteMetricsContainer{
38763963
registry: storeRegistry,
38773964
},
@@ -4212,6 +4299,7 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
42124299
ClosedTimestampPolicyChange: metric.NewCounter(metaClosedTimestampPolicyChange),
42134300
ClosedTimestampLatencyInfoMissing: metric.NewCounter(metaClosedTimestampLatencyInfoMissing),
42144301
}
4302+
42154303
sm.categoryIterMetrics.init(storeRegistry)
42164304

42174305
storeRegistry.AddMetricStruct(sm)
@@ -4378,6 +4466,15 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
43784466
// ratio; we estimate it from the data we do have.
43794467
sm.CompressionOverallCR.Update(overall.CompressionRatio())
43804468

4469+
sm.BytesCompressedL5Values.Update(int64(m.CompressionCounters.LogicalBytesCompressed.L5.ValueBlocks))
4470+
sm.BytesCompressedL5Data.Update(int64(m.CompressionCounters.LogicalBytesCompressed.L5.DataBlocks))
4471+
sm.BytesCompressedL6Values.Update(int64(m.CompressionCounters.LogicalBytesCompressed.L6.ValueBlocks))
4472+
sm.BytesCompressedL6Data.Update(int64(m.CompressionCounters.LogicalBytesCompressed.L6.DataBlocks))
4473+
sm.BytesDecompressedL5Values.Update(int64(m.CompressionCounters.LogicalBytesDecompressed.L5.ValueBlocks))
4474+
sm.BytesDecompressedL5Data.Update(int64(m.CompressionCounters.LogicalBytesDecompressed.L5.DataBlocks))
4475+
sm.BytesDecompressedL6Values.Update(int64(m.CompressionCounters.LogicalBytesDecompressed.L6.ValueBlocks))
4476+
sm.BytesDecompressedL6Data.Update(int64(m.CompressionCounters.LogicalBytesDecompressed.L6.DataBlocks))
4477+
43814478
sm.categoryIterMetrics.update(m.CategoryStats)
43824479
sm.categoryDiskWriteMetrics.update(m.DiskWriteStats)
43834480

‎pkg/roachprod/agents/opentelemetry/cockroachdb_metrics.go‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2283,6 +2283,14 @@ var cockroachdbMetrics = map[string]string{
22832283
"storage_batch_commit_wal_rotation_duration": "storage.batch_commit.wal_rotation.duration",
22842284
"storage_block_load_active": "storage.block_load.active",
22852285
"storage_block_load_queued": "storage.block_load.queued",
2286+
"storage_bytes_compressed_l5_data": "storage.bytes_compressed.l5.data",
2287+
"storage_bytes_compressed_l5_values": "storage.bytes_compressed.l5.values",
2288+
"storage_bytes_compressed_l6_data": "storage.bytes_compressed.l6.data",
2289+
"storage_bytes_compressed_l6_values": "storage.bytes_compressed.l6.values",
2290+
"storage_bytes_decompressed_l5_data": "storage.bytes_decompressed.l5.data",
2291+
"storage_bytes_decompressed_l5_values": "storage.bytes_decompressed.l5.values",
2292+
"storage_bytes_decompressed_l6_data": "storage.bytes_decompressed.l6.data",
2293+
"storage_bytes_decompressed_l6_values": "storage.bytes_decompressed.l6.values",
22862294
"storage_category_pebble_manifest_bytes_written": "storage.category_pebble_manifest.bytes_written",
22872295
"storage_category_pebble_wal_bytes_written": "storage.category_pebble_wal.bytes_written",
22882296
"storage_category_unspecified_bytes_written": "storage.category_unspecified.bytes_written",

‎pkg/util/metric/metric.go‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,7 @@ const (
4848
LabelCertificateType = "certificate_type"
4949
LabelName = "name"
5050
LabelType = "type"
51+
LabelLevel = "level"
5152
)
5253

5354
type LabelConfig uint64

0 commit comments

Comments
 (0)