*: allow compression type to be inferred from initial stats load

Currently, the compression type for a table is known when the SST is new
\- i.e. flushed, compacted, ingested. However, when opening an existing
DB, the compression types will show as "unknown".

Allow the async, background stats loading job that runs on DB open to
infer the compression type of existing tables by lifting the compression
field from `sstable.Properties` up into `sstable.CommonProperties`. This
allows `loadTableStats` to use the field and populate the
`manifest.TableStats` struct.

Author: nicktrav

metrics_test.go

@@ -163,6 +163,17 @@ func TestMetrics(t *testing.T) {
 		if createOnSharedLower {
 			require.NoError(t, d.SetCreatorID(1))
 		}
+		if reopen {
+			// Stats population is eventually consistent, and happens in the
+			// background when a DB is re-opened. To avoid races, wait synchronously
+			// for all tables to have their stats fully populated, which requires
+			// opening each SST.
+			d.mu.Lock()
+			for !d.mu.tableStats.loadedInitial {
+				d.mu.tableStats.cond.Wait()
+			}
+			d.mu.Unlock()
+		}
 		iters = make(map[string]*Iterator)
 		closeFunc = func() {
 			for _, i := range iters {

sstable/properties.go

@@ -99,6 +99,10 @@ type CommonProperties struct {
 	NumRangeKeySets uint64 `prop:"pebble.num.range-key-sets"`
 	// Total size of value blocks and value index block. Only serialized if > 0.
 	ValueBlocksSize uint64 `prop:"pebble.value-blocks.size"`
+	// The compression algorithm used to compress blocks.
+	CompressionName string `prop:"rocksdb.compression"`
+	// The compression options used to compress blocks.
+	CompressionOptions string `prop:"rocksdb.compression_options"`
 }
 
 // String is only used for testing purposes.
@@ -127,10 +131,6 @@ type Properties struct {
 
 	// The name of the comparer used in this table.
 	ComparerName string `prop:"rocksdb.comparator"`
-	// The compression algorithm used to compress blocks.
-	CompressionName string `prop:"rocksdb.compression"`
-	// The compression options used to compress blocks.
-	CompressionOptions string `prop:"rocksdb.compression_options"`
 	// The total size of all data blocks.
 	DataSize uint64 `prop:"rocksdb.data.size"`
 	// The name of the filter policy used in this table. Empty if no filter

sstable/properties_test.go

@@ -21,15 +21,15 @@ import (
 func TestPropertiesLoad(t *testing.T) {
 	expected := Properties{
 		CommonProperties: CommonProperties{
-			NumEntries:        1727,
-			NumDeletions:      17,
-			NumRangeDeletions: 17,
-			RawKeySize:        23938,
-			RawValueSize:      1912,
+			NumEntries:         1727,
+			NumDeletions:       17,
+			NumRangeDeletions:  17,
+			RawKeySize:         23938,
+			RawValueSize:       1912,
+			CompressionName:    "Snappy",
+			CompressionOptions: "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ",
 		},
 		ComparerName:           "leveldb.BytewiseComparator",
-		CompressionName:        "Snappy",
-		CompressionOptions:     "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ",
 		DataSize:               13913,
 		IndexSize:              325,
 		MergerName:             "nullptr",
@@ -57,17 +57,17 @@ func TestPropertiesLoad(t *testing.T) {
 
 var testProps = Properties{
 	CommonProperties: CommonProperties{
-		NumDeletions:      15,
-		NumEntries:        16,
-		NumRangeDeletions: 18,
-		NumRangeKeyDels:   19,
-		NumRangeKeySets:   20,
-		RawKeySize:        25,
-		RawValueSize:      26,
+		NumDeletions:       15,
+		NumEntries:         16,
+		NumRangeDeletions:  18,
+		NumRangeKeyDels:    19,
+		NumRangeKeySets:    20,
+		RawKeySize:         25,
+		RawValueSize:       26,
+		CompressionName:    "compression name",
+		CompressionOptions: "compression option",
 	},
 	ComparerName:           "comparator name",
-	CompressionName:        "compression name",
-	CompressionOptions:     "compression option",
 	DataSize:               3,
 	FilterPolicyName:       "filter policy name",
 	FilterSize:             5,
@@ -93,16 +93,16 @@ func TestPropertiesSave(t *testing.T) {
 	expected := &Properties{}
 	*expected = testProps
 
-	check1 := func(expected *Properties) {
+	check1 := func(e *Properties) {
 		// Check that we can save properties and read them back.
 		var w rawBlockWriter
 		w.restartInterval = propertiesBlockRestartInterval
-		expected.save(TableFormatPebblev2, &w)
+		e.save(TableFormatPebblev2, &w)
 		var props Properties
 
 		require.NoError(t, props.load(w.finish(), 0, make(map[string]struct{})))
 		props.Loaded = nil
-		if diff := pretty.Diff(*expected, props); diff != nil {
+		if diff := pretty.Diff(*e, props); diff != nil {
 			t.Fatalf("%s", strings.Join(diff, "\n"))
 		}
 	}
@@ -116,6 +116,7 @@ func TestPropertiesSave(t *testing.T) {
 		if props.IndexPartitions == 0 {
 			props.TopLevelIndexSize = 0
 		}
+		props.Loaded = nil
 		check1(&props)
 	}
 }

table_stats.go

@@ -323,6 +323,7 @@ func (d *DB) loadTableStats(
 			// picking.
 			stats.NumRangeKeySets = props.NumRangeKeySets
 			stats.ValueBlocksSize = props.ValueBlocksSize
+			stats.CompressionType = sstable.CompressionFromString(props.CompressionName)
 			return
 		})
 	if err != nil {

testdata/metrics

@@ -820,7 +820,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 0B
-Compression types: unknown: 1
+Compression types: snappy: 1
 Block cache: 1 entries (440B)  hit rate: 0.0%
 Table cache: 1 entries (760B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -867,7 +867,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 0B
-Compression types: snappy: 1 unknown: 1
+Compression types: snappy: 2
 Block cache: 6 entries (996B)  hit rate: 0.0%
 Table cache: 1 entries (760B)  hit rate: 50.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -915,7 +915,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 589B
-Compression types: snappy: 2 unknown: 1
+Compression types: snappy: 3
 Block cache: 6 entries (996B)  hit rate: 0.0%
 Table cache: 1 entries (760B)  hit rate: 50.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%
@@ -954,7 +954,7 @@ Zombie tables: 0 (0B, local: 0B)
 Backing tables: 0 (0B)
 Virtual tables: 0 (0B)
 Local tables size: 589B
-Compression types: unknown: 3
+Compression types: snappy: 3
 Block cache: 0 entries (0B)  hit rate: 0.0%
 Table cache: 0 entries (0B)  hit rate: 0.0%
 Secondary cache: 0 entries (0B)  hit rate: 0.0%

testdata/table_stats

@@ -602,9 +602,9 @@ rocksdb:
   rocksdb.raw.value.size: 2
   rocksdb.deleted.keys: 1
   rocksdb.num.range-deletions: 0
-  rocksdb.comparator: pebble.internal.testkeys
   rocksdb.compression: Snappy
   rocksdb.compression_options: window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; 
+  rocksdb.comparator: pebble.internal.testkeys
   rocksdb.data.size: 53
   rocksdb.filter.size: 0
   rocksdb.index.size: 27
@@ -710,9 +710,9 @@ rocksdb:
   rocksdb.raw.value.size: 3
   rocksdb.deleted.keys: 0
   rocksdb.num.range-deletions: 0
-  rocksdb.comparator: pebble.internal.testkeys
   rocksdb.compression: Snappy
   rocksdb.compression_options: window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; 
+  rocksdb.comparator: pebble.internal.testkeys
   rocksdb.data.size: 47
   rocksdb.filter.size: 0
   rocksdb.index.size: 27
@@ -832,9 +832,9 @@ rocksdb:
   rocksdb.raw.value.size: 1
   rocksdb.deleted.keys: 1
   rocksdb.num.range-deletions: 1
-  rocksdb.comparator: pebble.internal.testkeys
   rocksdb.compression: Snappy
   rocksdb.compression_options: window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; 
+  rocksdb.comparator: pebble.internal.testkeys
   rocksdb.data.size: 13
   rocksdb.filter.size: 0
   rocksdb.index.size: 29