block: improve adaptive compression profiles

We allow different adaptive ratios for data and value blocks and
improve the profiles.

Author: RaduBerinde

sstable/block/compression.go

@@ -29,13 +29,20 @@ type CompressionProfile struct {
 	//
 	// Note that MinLZ is only supported with table formats v6+. Older formats
 	// fall back to Snappy.
-	DataBlocks  compression.Setting
-	ValueBlocks compression.Setting
+	DataBlocks  CompressionSetting
+	ValueBlocks CompressionSetting
 	OtherBlocks compression.Setting
 
 	// Blocks that are reduced by less than this percentage are stored
 	// uncompressed.
 	MinReductionPercent uint8
+}
+
+// CompressionSetting is a compression setting for value or data blocks. It
+// contains a compression.Setting and an optional percentage which enables
+// adaptive compression.
+type CompressionSetting struct {
+	compression.Setting
 
 	// AdaptiveReductionCutoffPercent (when set to a non-zero value) enables
 	// adaptive compressors for data and value blocks which fall back to the
@@ -45,6 +52,20 @@ type CompressionProfile struct {
 	AdaptiveReductionCutoffPercent uint8
 }
 
+// SimpleCompressionSetting returns a CompressionSetting that always uses the
+// given compression.
+func SimpleCompressionSetting(s compression.Setting) CompressionSetting {
+	return CompressionSetting{Setting: s}
+}
+
+// AdaptiveCompressionSetting returns a CompressionSetting that adaptively
+// chooses between the enclosed setting or the "other blocks" setting.
+func AdaptiveCompressionSetting(
+	s compression.Setting, reductionCutoffPercent uint8,
+) CompressionSetting {
+	return CompressionSetting{Setting: s, AdaptiveReductionCutoffPercent: reductionCutoffPercent}
+}
+
 // UsesMinLZ returns true if the profile uses the MinLZ compression algorithm
 // (for any block kind).
 func (p *CompressionProfile) UsesMinLZ() bool {
@@ -68,33 +89,32 @@ var (
 	// FastCompression automatically chooses between Snappy/MinLZ1 and Zstd1 for
 	// sstable and blob file value blocks.
 	FastCompression = registerCompressionProfile(CompressionProfile{
-		Name:                           "Fast",
-		DataBlocks:                     fastestCompression,
-		ValueBlocks:                    compression.ZstdLevel1,
-		OtherBlocks:                    fastestCompression,
-		MinReductionPercent:            10,
-		AdaptiveReductionCutoffPercent: 30,
+		Name:                "Fast",
+		DataBlocks:          SimpleCompressionSetting(fastestCompression),
+		ValueBlocks:         AdaptiveCompressionSetting(compression.ZstdLevel1, 30),
+		OtherBlocks:         fastestCompression,
+		MinReductionPercent: 10,
 	})
 
 	// BalancedCompression automatically chooses between Snappy/MinLZ1 and Zstd1
 	// for data and value blocks.
 	BalancedCompression = registerCompressionProfile(CompressionProfile{
-		Name:                           "Balanced",
-		DataBlocks:                     compression.ZstdLevel1,
-		ValueBlocks:                    compression.ZstdLevel1,
-		OtherBlocks:                    fastestCompression,
-		MinReductionPercent:            5,
-		AdaptiveReductionCutoffPercent: 15,
+		Name:                "Balanced",
+		DataBlocks:          AdaptiveCompressionSetting(compression.ZstdLevel1, 30),
+		ValueBlocks:         AdaptiveCompressionSetting(compression.ZstdLevel1, 15),
+		OtherBlocks:         fastestCompression,
+		MinReductionPercent: 5,
 	})
 
 	// GoodCompression uses Zstd1 for data and value blocks.
+	//
+	// Note: in practice, we have observed very little size benefit to using
+	// higher zstd levels like ZstdLevel3 (while paying a significant compression
+	// performance cost).
 	GoodCompression = registerCompressionProfile(CompressionProfile{
-		Name: "Good",
-		// In practice, we have observed very little size benefit to using higher
-		// zstd levels like ZstdLevel3 while paying a significant compression
-		// performance cost.
-		DataBlocks:          compression.ZstdLevel1,
-		ValueBlocks:         compression.ZstdLevel1,
+		Name:                "Good",
+		DataBlocks:          SimpleCompressionSetting(compression.ZstdLevel1),
+		ValueBlocks:         SimpleCompressionSetting(compression.ZstdLevel1),
 		OtherBlocks:         fastestCompression,
 		MinReductionPercent: 3,
 	})
@@ -119,8 +139,8 @@ var fastestCompression = func() compression.Setting {
 func simpleCompressionProfile(name string, setting compression.Setting) *CompressionProfile {
 	return registerCompressionProfile(CompressionProfile{
 		Name:                name,
-		DataBlocks:          setting,
-		ValueBlocks:         setting,
+		DataBlocks:          SimpleCompressionSetting(setting),
+		ValueBlocks:         SimpleCompressionSetting(setting),
 		OtherBlocks:         setting,
 		MinReductionPercent: 12,
 	})

sstable/block/compressor.go

@@ -40,20 +40,20 @@ func MakeCompressor(profile *CompressionProfile) Compressor {
 }
 
 func maybeAdaptiveCompressor(
-	profile *CompressionProfile, setting compression.Setting,
+	profile *CompressionProfile, setting CompressionSetting,
 ) compression.Compressor {
-	if profile.AdaptiveReductionCutoffPercent != 0 && setting != profile.OtherBlocks {
+	if setting.AdaptiveReductionCutoffPercent != 0 && setting.Setting != profile.OtherBlocks {
 		params := compression.AdaptiveCompressorParams{
-			Slow:            setting,
+			Slow:            setting.Setting,
 			Fast:            profile.OtherBlocks,
-			ReductionCutoff: float64(profile.AdaptiveReductionCutoffPercent) * 0.01,
+			ReductionCutoff: float64(setting.AdaptiveReductionCutoffPercent) * 0.01,
 			SampleEvery:     10,
 			SampleHalfLife:  256 * 1024, // 256 KB
 			SamplingSeed:    rand.Uint64(),
 		}
 		return compression.NewAdaptiveCompressor(params)
 	}
-	return compression.GetCompressor(setting)
+	return compression.GetCompressor(setting.Setting)
 }
 
 // Close must be called when the Compressor is no longer needed.

sstable/block/compressor_test.go

@@ -25,8 +25,8 @@ func TestCompressor(t *testing.T) {
 	dst := make([]byte, 0, 1024)
 	for runs := 0; runs < 100; runs++ {
 		profile := &CompressionProfile{
-			DataBlocks:          settings[rand.IntN(len(settings))],
-			ValueBlocks:         settings[rand.IntN(len(settings))],
+			DataBlocks:          SimpleCompressionSetting(settings[rand.IntN(len(settings))]),
+			ValueBlocks:         SimpleCompressionSetting(settings[rand.IntN(len(settings))]),
 			OtherBlocks:         settings[rand.IntN(len(settings))],
 			MinReductionPercent: 0,
 		}

sstable/compressionanalyzer/buckets.go

@@ -99,49 +99,47 @@ func (c Compressibility) String() string {
 var Profiles = [...]*block.CompressionProfile{
 	{
 		Name:                "Snappy",
-		DataBlocks:          compression.SnappySetting,
-		ValueBlocks:         compression.SnappySetting,
+		DataBlocks:          block.SimpleCompressionSetting(compression.SnappySetting),
+		ValueBlocks:         block.SimpleCompressionSetting(compression.SnappySetting),
 		OtherBlocks:         compression.SnappySetting,
 		MinReductionPercent: 0,
 	},
 
 	{
 		Name:                "MinLZ1",
-		DataBlocks:          compression.MinLZFastest,
-		ValueBlocks:         compression.MinLZFastest,
+		DataBlocks:          block.SimpleCompressionSetting(compression.MinLZFastest),
+		ValueBlocks:         block.SimpleCompressionSetting(compression.MinLZFastest),
 		OtherBlocks:         compression.MinLZFastest,
 		MinReductionPercent: 0,
 	},
 	{
 		Name:                "Zstd1",
-		DataBlocks:          compression.ZstdLevel1,
-		ValueBlocks:         compression.ZstdLevel1,
+		DataBlocks:          block.SimpleCompressionSetting(compression.ZstdLevel1),
+		ValueBlocks:         block.SimpleCompressionSetting(compression.ZstdLevel1),
 		OtherBlocks:         compression.ZstdLevel1,
 		MinReductionPercent: 0,
 	},
 
 	{
-		Name:                           "Auto1/30",
-		DataBlocks:                     compression.ZstdLevel1,
-		ValueBlocks:                    compression.ZstdLevel1,
-		OtherBlocks:                    compression.MinLZFastest,
-		AdaptiveReductionCutoffPercent: 30,
-		MinReductionPercent:            0,
+		Name:                "Auto1/30",
+		DataBlocks:          block.AdaptiveCompressionSetting(compression.ZstdLevel1, 30),
+		ValueBlocks:         block.AdaptiveCompressionSetting(compression.ZstdLevel1, 30),
+		OtherBlocks:         compression.MinLZFastest,
+		MinReductionPercent: 0,
 	},
 
 	{
-		Name:                           "Auto1/15",
-		DataBlocks:                     compression.ZstdLevel1,
-		ValueBlocks:                    compression.ZstdLevel1,
-		OtherBlocks:                    compression.MinLZFastest,
-		AdaptiveReductionCutoffPercent: 15,
-		MinReductionPercent:            0,
+		Name:                "Auto1/15",
+		DataBlocks:          block.AdaptiveCompressionSetting(compression.ZstdLevel1, 15),
+		ValueBlocks:         block.AdaptiveCompressionSetting(compression.ZstdLevel1, 15),
+		OtherBlocks:         compression.MinLZFastest,
+		MinReductionPercent: 0,
 	},
 
 	{
 		Name:                "Zstd3",
-		DataBlocks:          compression.ZstdLevel3,
-		ValueBlocks:         compression.ZstdLevel3,
+		DataBlocks:          block.SimpleCompressionSetting(compression.ZstdLevel3),
+		ValueBlocks:         block.SimpleCompressionSetting(compression.ZstdLevel3),
 		OtherBlocks:         compression.ZstdLevel3,
 		MinReductionPercent: 0,
 	},