*: add metric for sstable compression types

itsbilal · nicktrav · commit 30d2cf60e8f5 · 2024-05-17T15:56:28.000-07:00
Previously, we didn't have a metric tracking the compression type(s) of sstables. This change adds that, as a table stat that's collected through the LevelMetadata as an annotator field. This unfortunately had to be a table stat as the compression type isn't known until we open a reader and look at the properties block. Pebble side of cockroachdb/cockroach#123952.
diff --git a/db.go b/db.go
@@ -2015,6 +2015,11 @@ func (d *DB) Metrics() *Metrics {
 	}
 	for i := 0; i < numLevels; i++ {
 		metrics.Levels[i].Additional.ValueBlocksSize = valueBlocksSizeForLevel(vers, i)
+		unknown, snappy, none, zstd := compressionTypesForLevel(vers, i)
+		metrics.Table.CompressedCountUnknown += int64(unknown)
+		metrics.Table.CompressedCountSnappy += int64(snappy)
+		metrics.Table.CompressedCountZstd += int64(zstd)
+		metrics.Table.CompressedCountNone += int64(none)
 	}
 
 	d.mu.Unlock()
diff --git a/ingest_test.go b/ingest_test.go
@@ -172,6 +172,7 @@ func TestIngestLoadRand(t *testing.T) {
 			},
 			path: paths[i],
 		}
+		expected[i].fileMetadata.Stats.CompressionType = sstable.SnappyCompression
 		expected[i].StatsMarkValid()
 
 		func() {
diff --git a/internal/manifest/version.go b/internal/manifest/version.go
@@ -72,6 +72,8 @@ type TableStats struct {
 	RangeDeletionsBytesEstimate uint64
 	// Total size of value blocks and value index block.
 	ValueBlocksSize uint64
+	// CompressionType is the compression type of the table.
+	CompressionType sstable.Compression
 }
 
 // boundType represents the type of key (point or range) present as the smallest
diff --git a/metrics.go b/metrics.go
@@ -265,6 +265,16 @@ type Metrics struct {
 		BackingTableCount uint64
 		// The sum of the sizes of the BackingTableCount sstables that are backing virtual tables.
 		BackingTableSize uint64
+		// The number of sstables that are compressed with an unknown compression
+		// algorithm.
+		CompressedCountUnknown int64
+		// The number of sstables that are compressed with the default compression
+		// algorithm, snappy.
+		CompressedCountSnappy int64
+		// The number of sstables that are compressed with zstd.
+		CompressedCountZstd int64
+		// The number of sstables that are uncompressed.
+		CompressedCountNone int64
 
 		// Local file sizes.
 		Local struct {
@@ -598,6 +608,20 @@ func (m *Metrics) SafeFormat(w redact.SafePrinter, _ rune) {
 		redact.Safe(m.NumVirtual()),
 		humanize.Bytes.Uint64(m.VirtualSize()))
 	w.Printf("Local tables size: %s\n", humanize.Bytes.Uint64(m.Table.Local.LiveSize))
+	w.SafeString("Compression types:")
+	if count := m.Table.CompressedCountSnappy; count > 0 {
+		w.Printf(" snappy: %d", redact.Safe(count))
+	}
+	if count := m.Table.CompressedCountZstd; count > 0 {
+		w.Printf(" zstd: %d", redact.Safe(count))
+	}
+	if count := m.Table.CompressedCountNone; count > 0 {
+		w.Printf(" none: %d", redact.Safe(count))
+	}
+	if count := m.Table.CompressedCountUnknown; count > 0 {
+		w.Printf(" unknown: %d", redact.Safe(count))
+	}
+	w.Print("\n")
 
 	formatCacheMetrics := func(m *CacheMetrics, name redact.SafeString) {
 		w.Printf("%s: %s entries (%s)  hit rate: %.1f%%\n",
diff --git a/sstable/options.go b/sstable/options.go
@@ -47,6 +47,23 @@ func (c Compression) String() string {
 	}
 }
 
+// CompressionFromString returns an sstable.Compression from its
+// string representation. Inverse of c.String() above.
+func CompressionFromString(s string) Compression {
+	switch s {
+	case "Default":
+		return DefaultCompression
+	case "NoCompression":
+		return NoCompression
+	case "Snappy":
+		return SnappyCompression
+	case "ZSTD":
+		return ZstdCompression
+	default:
+		return DefaultCompression
+	}
+}
+
 // FilterType exports the base.FilterType type.
 type FilterType = base.FilterType
 
diff --git a/table_stats.go b/table_stats.go
@@ -677,6 +677,7 @@ func maybeSetStatsFromProperties(meta physicalMeta, props *sstable.Properties) b
 	meta.Stats.PointDeletionsBytesEstimate = pointEstimate
 	meta.Stats.RangeDeletionsBytesEstimate = 0
 	meta.Stats.ValueBlocksSize = props.ValueBlocksSize
+	meta.Stats.CompressionType = sstable.CompressionFromString(props.CompressionName)
 	meta.StatsMarkValid()
 	return true
 }
@@ -1127,3 +1128,66 @@ func valueBlocksSizeForLevel(v *version, level int) (count uint64) {
 	}
 	return *v.Levels[level].Annotation(valueBlocksSizeAnnotator{}).(*uint64)
 }
+
+// compressionTypeAnnotator implements manifest.Annotator, annotating B-tree
+// nodes with the compression type of the file. Its annotation type is a
+// *compressionTypes. The compression type may change once a table's stats are
+// loaded asynchronously, so its values are marked as cacheable only if a file's
+// stats have been loaded.
+type compressionTypeAnnotator struct{}
+
+type compressionTypes struct {
+	snappy, zstd, none, unknown uint64
+}
+
+var _ manifest.Annotator = compressionTypeAnnotator{}
+
+func (a compressionTypeAnnotator) Zero(dst interface{}) interface{} {
+	if dst == nil {
+		return new(compressionTypes)
+	}
+	v := dst.(*compressionTypes)
+	*v = compressionTypes{}
+	return v
+}
+
+func (a compressionTypeAnnotator) Accumulate(
+	f *fileMetadata, dst interface{},
+) (v interface{}, cacheOK bool) {
+	vptr := dst.(*compressionTypes)
+	switch f.Stats.CompressionType {
+	case sstable.SnappyCompression:
+		vptr.snappy++
+	case sstable.ZstdCompression:
+		vptr.zstd++
+	case sstable.NoCompression:
+		vptr.none++
+	default:
+		vptr.unknown++
+	}
+	return vptr, f.StatsValid()
+}
+
+func (a compressionTypeAnnotator) Merge(src interface{}, dst interface{}) interface{} {
+	srcV := src.(*compressionTypes)
+	dstV := dst.(*compressionTypes)
+	dstV.snappy = dstV.snappy + srcV.snappy
+	dstV.zstd = dstV.zstd + srcV.zstd
+	dstV.none = dstV.none + srcV.none
+	dstV.unknown = dstV.unknown + srcV.unknown
+	return dstV
+}
+
+// compressionTypesForLevel returns the count of sstables by compression type
+// used for a level in the LSM. Sstables with compression type snappy or zstd
+// are returned, while others are ignored.
+func compressionTypesForLevel(v *version, level int) (unknown, snappy, none, zstd uint64) {
+	if v.Levels[level].Empty() {
+		return
+	}
+	compression := v.Levels[level].Annotation(compressionTypeAnnotator{}).(*compressionTypes)
+	if compression == nil {
+		return
+	}
+	return compression.unknown, compression.snappy, compression.none, compression.zstd
+}
diff --git a/testdata/event_listener b/testdata/event_listener
@@ -228,6 +228,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 1.7KB
+Compression types: snappy: 3
 Block cache: 6 entries (970B)  hit rate: 0.0%
 Table cache: 1 entries (760B)  hit rate: 40.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -328,6 +329,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 3.5KB
+Compression types: snappy: 6
 Block cache: 12 entries (1.9KB)  hit rate: 7.7%
 Table cache: 1 entries (760B)  hit rate: 50.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
diff --git a/testdata/ingest b/testdata/ingest
@@ -52,6 +52,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 569B
+Compression types: snappy: 1
 Block cache: 6 entries (945B)  hit rate: 30.8%
 Table cache: 1 entries (760B)  hit rate: 50.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
diff --git a/testdata/metrics b/testdata/metrics
@@ -21,6 +21,7 @@ Zombie tables: 16 (15B, local: 30B)
 Backing tables: 1 (2.0MB)
 Virtual tables: 2807 (2.8KB)
 Local tables size: 28B
+Compression types:
 Block cache: 2 entries (1B)  hit rate: 42.9%
 Table cache: 18 entries (17B)  hit rate: 48.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -72,6 +73,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 589B
+Compression types: snappy: 1
 Block cache: 3 entries (484B)  hit rate: 0.0%
 Table cache: 1 entries (760B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -127,6 +129,7 @@ Zombie tables: 2 (1.2KB, local: 1.2KB)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 595B
+Compression types: snappy: 1
 Block cache: 5 entries (946B)  hit rate: 33.3%
 Table cache: 2 entries (1.5KB)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -169,6 +172,7 @@ Zombie tables: 2 (1.2KB, local: 1.2KB)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 595B
+Compression types: snappy: 1
 Block cache: 5 entries (946B)  hit rate: 33.3%
 Table cache: 2 entries (1.5KB)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -208,6 +212,7 @@ Zombie tables: 1 (589B, local: 589B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 595B
+Compression types: snappy: 1
 Block cache: 3 entries (484B)  hit rate: 33.3%
 Table cache: 1 entries (760B)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -251,6 +256,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 595B
+Compression types: snappy: 1
 Block cache: 0 entries (0B)  hit rate: 33.3%
 Table cache: 0 entries (0B)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -321,6 +327,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 2.6KB
+Compression types: snappy: 4
 Block cache: 0 entries (0B)  hit rate: 33.3%
 Table cache: 0 entries (0B)  hit rate: 66.7%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -375,6 +382,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 2.0KB
+Compression types: snappy: 3
 Block cache: 0 entries (0B)  hit rate: 14.3%
 Table cache: 0 entries (0B)  hit rate: 58.3%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -478,6 +486,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 4.3KB
+Compression types: snappy: 7
 Block cache: 12 entries (1.9KB)  hit rate: 16.7%
 Table cache: 1 entries (760B)  hit rate: 60.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -540,6 +549,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 6.1KB
+Compression types: snappy: 10
 Block cache: 12 entries (1.9KB)  hit rate: 16.7%
 Table cache: 1 entries (760B)  hit rate: 60.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -616,6 +626,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 2 (1.2KB)
 Virtual tables: 2 (102B)
 Local tables size: 6.7KB
+Compression types: snappy: 9 unknown: 2
 Block cache: 0 entries (0B)  hit rate: 0.0%
 Table cache: 0 entries (0B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -717,6 +728,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 3.8KB
+Compression types: snappy: 6
 Block cache: 0 entries (0B)  hit rate: 0.0%
 Table cache: 0 entries (0B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -770,6 +782,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 604B
+Compression types: snappy: 1
 Block cache: 0 entries (0B)  hit rate: 0.0%
 Table cache: 0 entries (0B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -807,6 +820,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 0B
+Compression types: unknown: 1
 Block cache: 1 entries (440B)  hit rate: 0.0%
 Table cache: 1 entries (760B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -853,6 +867,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 0B
+Compression types: snappy: 1 unknown: 1
 Block cache: 6 entries (996B)  hit rate: 0.0%
 Table cache: 1 entries (760B)  hit rate: 50.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -900,6 +915,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 589B
+Compression types: snappy: 2 unknown: 1
 Block cache: 6 entries (996B)  hit rate: 0.0%
 Table cache: 1 entries (760B)  hit rate: 50.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -938,6 +954,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 589B
+Compression types: unknown: 3
 Block cache: 0 entries (0B)  hit rate: 0.0%
 Table cache: 0 entries (0B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -975,6 +992,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 603B
+Compression types: snappy: 1
 Block cache: 0 entries (0B)  hit rate: 0.0%
 Table cache: 0 entries (0B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
diff --git a/tool/testdata/db_lsm b/tool/testdata/db_lsm
@@ -31,6 +31,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 709B
+Compression types: unknown: 1
 Block cache: 0 entries (0B)  hit rate: 0.0%
 Table cache: 0 entries (0B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -64,6 +65,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 709B
+Compression types: unknown: 1
 Block cache: 0 entries (0B)  hit rate: 0.0%
 Table cache: 0 entries (0B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%

Original file line number	Diff line number	Diff line change
`@@ -2015,6 +2015,11 @@ func (d DB) Metrics() Metrics {`
`2015`	`2015`	`}`
`2016`	`2016`	`for i := 0; i < numLevels; i++ {`
`2017`	`2017`	`metrics.Levels[i].Additional.ValueBlocksSize = valueBlocksSizeForLevel(vers, i)`
	`2018`	`+ unknown, snappy, none, zstd := compressionTypesForLevel(vers, i)`
	`2019`	`+ metrics.Table.CompressedCountUnknown += int64(unknown)`
	`2020`	`+ metrics.Table.CompressedCountSnappy += int64(snappy)`
	`2021`	`+ metrics.Table.CompressedCountZstd += int64(zstd)`
	`2022`	`+ metrics.Table.CompressedCountNone += int64(none)`
`2018`	`2023`	`}`
`2019`	`2024`
`2020`	`2025`	`d.mu.Unlock()`
Original file line number	Diff line number	Diff line change
`@@ -172,6 +172,7 @@ func TestIngestLoadRand(t *testing.T) {`
`172`	`172`	`},`
`173`	`173`	`path: paths[i],`
`174`	`174`	`}`
	`175`	`+ expected[i].fileMetadata.Stats.CompressionType = sstable.SnappyCompression`
`175`	`176`	`expected[i].StatsMarkValid()`
`176`	`177`
`177`	`178`	`func() {`
Original file line number	Diff line number	Diff line change
`@@ -72,6 +72,8 @@ type TableStats struct {`
`72`	`72`	`RangeDeletionsBytesEstimate uint64`
`73`	`73`	`// Total size of value blocks and value index block.`
`74`	`74`	`ValueBlocksSize uint64`
	`75`	`+ // CompressionType is the compression type of the table.`
	`76`	`+ CompressionType sstable.Compression`
`75`	`77`	`}`
`76`	`78`
`77`	`79`	`// boundType represents the type of key (point or range) present as the smallest`