Merge #147721 #147728

147721: storage: use MinLZ by default only on amd64 r=RaduBerinde a=RaduBerinde

On other platforms the MinLZ implementation is much slower so it's
safer to just stick to Snappy.

Epic: none
Release note: None

147728: kvserver: add new value separation timeseries r=jbowens a=jbowens

Add new timeseries metrics for value separation.

Epic: CRDB-20379
Release note (ops change): Introduces new timeseries metrics for observing the behavior of storage engine value separation.

Co-authored-by: Radu Berinde <[email protected]>
Co-authored-by: Jackson Owens <[email protected]>

Author: 3 people

docs/generated/metrics/metrics.yaml

@@ -16913,6 +16913,38 @@ layers:
       unit: BYTES
       aggregation: AVG
       derivative: NONE
+    - name: storage.value_separation.blob_files.count
+      exported_name: storage_value_separation_blob_files_count
+      description: The number of blob files that are used to store separated values within the storage engine.
+      y_axis_label: Files
+      type: GAUGE
+      unit: COUNT
+      aggregation: AVG
+      derivative: NONE
+    - name: storage.value_separation.blob_files.size
+      exported_name: storage_value_separation_blob_files_size
+      description: The size of the physical blob files that are used to store separated values within the storage engine. This sum is the physical post-compression sum of value_bytes.referenced and value_bytes.unreferenced.
+      y_axis_label: Bytes
+      type: GAUGE
+      unit: BYTES
+      aggregation: AVG
+      derivative: NONE
+    - name: storage.value_separation.value_bytes.referenced
+      exported_name: storage_value_separation_value_bytes_referenced
+      description: The size of storage engine value bytes (pre-compression) that are stored separately in blob files and referenced by a live sstable.
+      y_axis_label: Bytes
+      type: GAUGE
+      unit: BYTES
+      aggregation: AVG
+      derivative: NONE
+    - name: storage.value_separation.value_bytes.unreferenced
+      exported_name: storage_value_separation_value_bytes_unreferenced
+      description: The size of storage engine value bytes (pre-compression) that are stored separately in blob files and not referenced by any live sstable. These bytes are garbage that could be reclaimed by a compaction.
+      y_axis_label: Bytes
+      type: GAUGE
+      unit: BYTES
+      aggregation: AVG
+      derivative: NONE
     - name: storage.wal.bytes_in
       exported_name: storage_wal_bytes_in
       description: The number of logical bytes the storage engine has written to the WAL

pkg/kv/kvserver/metrics.go

@@ -2615,6 +2615,30 @@ Note that the measurement does not include the duration for replicating the eval
 		Measurement: "Flush Utilization",
 		Unit:        metric.Unit_PERCENT,
 	}
+	metaValueSeparationBytesReferenced = metric.Metadata{
+		Name:        "storage.value_separation.value_bytes.referenced",
+		Help:        "The size of storage engine value bytes (pre-compression) that are stored separately in blob files and referenced by a live sstable.",
+		Measurement: "Bytes",
+		Unit:        metric.Unit_BYTES,
+	}
+	metaValueSeparationBytesUnreferenced = metric.Metadata{
+		Name:        "storage.value_separation.value_bytes.unreferenced",
+		Help:        "The size of storage engine value bytes (pre-compression) that are stored separately in blob files and not referenced by any live sstable. These bytes are garbage that could be reclaimed by a compaction.",
+		Measurement: "Bytes",
+		Unit:        metric.Unit_BYTES,
+	}
+	metaValueSeparationBlobFileCount = metric.Metadata{
+		Name:        "storage.value_separation.blob_files.count",
+		Help:        "The number of blob files that are used to store separated values within the storage engine.",
+		Measurement: "Files",
+		Unit:        metric.Unit_COUNT,
+	}
+	metaValueSeparationBlobFileSize = metric.Metadata{
+		Name:        "storage.value_separation.blob_files.size",
+		Help:        "The size of the physical blob files that are used to store separated values within the storage engine. This sum is the physical post-compression sum of value_bytes.referenced and value_bytes.unreferenced.",
+		Measurement: "Bytes",
+		Unit:        metric.Unit_BYTES,
+	}
 	metaWALBytesWritten = metric.Metadata{
 		Name:        "storage.wal.bytes_written",
 		Help:        "The number of bytes the storage engine has written to the WAL",
@@ -2914,6 +2938,10 @@ type StoreMetrics struct {
 	SSTableCompressionNone            *metric.Gauge
 	categoryIterMetrics               pebbleCategoryIterMetricsContainer
 	categoryDiskWriteMetrics          pebbleCategoryDiskWriteMetricsContainer
+	ValueSeparationBytesReferenced    *metric.Gauge
+	ValueSeparationBytesUnreferenced  *metric.Gauge
+	ValueSeparationBlobFileCount      *metric.Gauge
+	ValueSeparationBlobFileSize       *metric.Gauge
 	WALBytesWritten                   *metric.Counter
 	WALBytesIn                        *metric.Counter
 	WALFailoverSwitchCount            *metric.Counter
@@ -3650,11 +3678,15 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		categoryDiskWriteMetrics: pebbleCategoryDiskWriteMetricsContainer{
 			registry: storeRegistry,
 		},
-		WALBytesWritten:              metric.NewCounter(metaWALBytesWritten),
-		WALBytesIn:                   metric.NewCounter(metaWALBytesIn),
-		WALFailoverSwitchCount:       metric.NewCounter(metaStorageWALFailoverSwitchCount),
-		WALFailoverPrimaryDuration:   metric.NewCounter(metaStorageWALFailoverPrimaryDuration),
-		WALFailoverSecondaryDuration: metric.NewCounter(metaStorageWALFailoverSecondaryDuration),
+		ValueSeparationBytesReferenced:   metric.NewGauge(metaValueSeparationBytesReferenced),
+		ValueSeparationBytesUnreferenced: metric.NewGauge(metaValueSeparationBytesUnreferenced),
+		ValueSeparationBlobFileCount:     metric.NewGauge(metaValueSeparationBlobFileCount),
+		ValueSeparationBlobFileSize:      metric.NewGauge(metaValueSeparationBlobFileSize),
+		WALBytesWritten:                  metric.NewCounter(metaWALBytesWritten),
+		WALBytesIn:                       metric.NewCounter(metaWALBytesIn),
+		WALFailoverSwitchCount:           metric.NewCounter(metaStorageWALFailoverSwitchCount),
+		WALFailoverPrimaryDuration:       metric.NewCounter(metaStorageWALFailoverPrimaryDuration),
+		WALFailoverSecondaryDuration:     metric.NewCounter(metaStorageWALFailoverSecondaryDuration),
 		WALFailoverWriteAndSyncLatency: metric.NewManualWindowHistogram(
 			metaStorageWALFailoverWriteAndSyncLatency,
 			pebble.FsyncLatencyBuckets,
@@ -4088,6 +4120,10 @@ func (sm *StoreMetrics) updateEngineMetrics(m storage.Metrics) {
 	sm.FlushableIngestTableCount.Update(int64(m.Flush.AsIngestTableCount))
 	sm.FlushableIngestTableSize.Update(int64(m.Flush.AsIngestBytes))
 	sm.IngestCount.Update(int64(m.Ingest.Count))
+	sm.ValueSeparationBytesReferenced.Update(int64(m.BlobFiles.ReferencedValueSize))
+	sm.ValueSeparationBytesUnreferenced.Update(int64(m.BlobFiles.ValueSize - m.BlobFiles.ReferencedValueSize))
+	sm.ValueSeparationBlobFileCount.Update(int64(m.BlobFiles.LiveCount))
+	sm.ValueSeparationBlobFileSize.Update(int64(m.BlobFiles.LiveSize))
 	// NB: `UpdateIfHigher` is used here since there is a race in pebble where
 	// sometimes the WAL is rotated but metrics are retrieved prior to the update
 	// to BytesIn to account for the previous WAL.

pkg/storage/pebble.go

@@ -12,6 +12,7 @@ import (
 	"math"
 	"os"
 	"path/filepath"
+	"runtime"
 	"sort"
 	"strconv"
 	"strings"
@@ -224,6 +225,19 @@ func RegisterCompressionAlgorithmClusterSetting(
 	)
 }
 
+var defaultCompressionAlgorithm = func() CompressionAlgorithm {
+	if runtime.GOARCH == "amd64" {
+		// We prefer MinLZ on amd64 because it is slightly superior to Snappy in
+		// almost all cases (both in terms of speed and compression ratio).
+		//
+		// Only amd64 has an optimized assembly MinLZ implementation; the Go
+		// implementation is significantly slower, especially when decompressing;
+		// see https://github.com/minio/minlz#protobuf-sample
+		return CompressionAlgorithmMinLZ
+	}
+	return CompressionAlgorithmSnappy
+}()
+
 // CompressionAlgorithmStorage determines the compression algorithm used to
 // compress data blocks when writing sstables for use in a Pebble store (written
 // directly, or constructed for ingestion on a remote store via AddSSTable).
@@ -232,7 +246,7 @@ func RegisterCompressionAlgorithmClusterSetting(
 var CompressionAlgorithmStorage = RegisterCompressionAlgorithmClusterSetting(
 	"storage.sstable.compression_algorithm",
 	`determines the compression algorithm to use when compressing sstable data blocks for use in a Pebble store;`,
-	CompressionAlgorithmMinLZ, // Default.
+	defaultCompressionAlgorithm,
 )
 
 // CompressionAlgorithmBackupStorage determines the compression algorithm used
@@ -242,7 +256,7 @@ var CompressionAlgorithmStorage = RegisterCompressionAlgorithmClusterSetting(
 var CompressionAlgorithmBackupStorage = RegisterCompressionAlgorithmClusterSetting(
 	"storage.sstable.compression_algorithm_backup_storage",
 	`determines the compression algorithm to use when compressing sstable data blocks for backup row data storage;`,
-	CompressionAlgorithmMinLZ, // Default.
+	defaultCompressionAlgorithm,
 )
 
 // CompressionAlgorithmBackupTransport determines the compression algorithm used
@@ -255,7 +269,7 @@ var CompressionAlgorithmBackupStorage = RegisterCompressionAlgorithmClusterSetti
 var CompressionAlgorithmBackupTransport = RegisterCompressionAlgorithmClusterSetting(
 	"storage.sstable.compression_algorithm_backup_transport",
 	`determines the compression algorithm to use when compressing sstable data blocks for backup transport;`,
-	CompressionAlgorithmMinLZ, // Default.
+	defaultCompressionAlgorithm,
 )
 
 func getCompressionAlgorithm(