storage: enable multilevel compactions by default with a cluster setting

Reenable multilevel compactions, with more constraints. ML compactions will run
according to a write amp heuristic, along with the following rules:
- At most 1 ML compaction is running at any time.
- The db must have >= 2 maximum compaction slots.

Closes: cockroachdb/pebble#4139

Author: xinhaoz

pkg/storage/pebble.go

@@ -167,6 +167,13 @@ var readaheadModeSpeculative = settings.RegisterEnumSetting(
 	},
 )
 
+var enableMultiLevelWriteAmpHeuristic = settings.RegisterBoolSetting(
+	settings.ApplicationLevel,
+	"storage.multi_level_compaction_write_amp_heuristic.enabled",
+	"enables multi-level compactions using the write amplification heuristic",
+	true,
+)
+
 // SSTableCompressionProfile is an enumeration of compression algorithms
 // available for compressing SSTables (e.g. for backup or transport).
 type SSTableCompressionProfile int64
@@ -564,11 +571,6 @@ func DefaultPebbleOptions() *pebble.Options {
 		policy.ValueStoragePolicy = pebble.ValueStorageLowReadLatency
 		return policy, lockTableEndKey, nil
 	}
-
-	// Disable multi-level compaction heuristic for now. See #134423
-	// for why this was disabled, and what needs to be changed to reenable it.
-	// This issue tracks re-enablement: https://github.com/cockroachdb/pebble/issues/4139
-	opts.Experimental.MultiLevelCompactionHeuristic = pebble.NoMultiLevel{}
 	opts.Experimental.UserKeyCategories = userKeyCategories
 
 	opts.Levels[0] = pebble.LevelOptions{
@@ -924,6 +926,14 @@ func newPebble(ctx context.Context, cfg engineConfig) (p *Pebble, err error) {
 			TargetGarbageRatio:    float64(valueSeparationCompactionGarbageThreshold.Get(&cfg.settings.SV)) / 100.0,
 		}
 	}
+	cfg.opts.Experimental.MultiLevelCompactionHeuristic = func() pebble.MultiLevelHeuristic {
+		if enableMultiLevelWriteAmpHeuristic.Get(&cfg.settings.SV) {
+			// Use the default write amp heuristic, which adds no propensity towards
+			// multi-level compactions and disallows multi-level compactions involving L0.
+			return pebble.OptionWriteAmpHeuristic()
+		}
+		return pebble.OptionNoMultiLevel()
+	}
 
 	auxDir := cfg.opts.FS.PathJoin(cfg.env.Dir, base.AuxiliaryDir)
 	if !cfg.env.IsReadOnly() {