storage: prefer fast compression for all local keys

Improve the span policy function to set the `PreferFastCompression`
flag for all local keys. Better compression for these keys won't save
a significant amount of space, but might affect performance (e.g. raft
logs, lock keys).

Epic: none
Release note: None

Author: RaduBerinde

pkg/storage/pebble.go

@@ -6,6 +6,7 @@
 package storage
 
 import (
+	"bytes"
 	"context"
 	"encoding/json"
 	"fmt"
@@ -464,17 +465,30 @@ func DefaultPebbleOptions() *pebble.Options {
 	// once.
 	opts.TargetByteDeletionRate = 128 << 20 // 128 MB
 	opts.Experimental.ShortAttributeExtractor = shortAttributeExtractorForValues
-	opts.Experimental.SpanPolicyFunc = pebble.MakeStaticSpanPolicyFunc(
-		cockroachkvs.Compare,
-		pebble.KeyRange{
-			Start: EncodeMVCCKey(MVCCKey{Key: keys.LocalRangeLockTablePrefix}),
-			End:   EncodeMVCCKey(MVCCKey{Key: keys.LocalRangeLockTablePrefix.PrefixEnd()}),
-		},
-		pebble.SpanPolicy{
-			DisableValueSeparationBySuffix: true,
-			ValueStoragePolicy:             pebble.ValueStorageLowReadLatency,
-		},
-	)
+
+	lockTableStartKey := EncodeMVCCKey(MVCCKey{Key: keys.LocalRangeLockTablePrefix})
+	lockTableEndKey := EncodeMVCCKey(MVCCKey{Key: keys.LocalRangeLockTablePrefix.PrefixEnd()})
+	localEndKey := EncodeMVCCKey(MVCCKey{Key: keys.LocalPrefix.PrefixEnd()})
+	opts.Experimental.SpanPolicyFunc = func(startKey []byte) (policy pebble.SpanPolicy, endKey []byte, _ error) {
+		if !bytes.HasPrefix(startKey, keys.LocalPrefix) {
+			return pebble.SpanPolicy{}, nil, nil
+		}
+		// Prefer fast compression for all local keys, since they shouldn't take up
+		// a significant part of the space.
+		policy.PreferFastCompression = true
+
+		// We also disable value separation for lock keys.
+		if cockroachkvs.Compare(startKey, lockTableEndKey) >= 0 {
+			return policy, localEndKey, nil
+		}
+		if cockroachkvs.Compare(startKey, lockTableStartKey) < 0 {
+			return policy, lockTableStartKey, nil
+		}
+		policy.DisableValueSeparationBySuffix = true
+		policy.ValueStoragePolicy = pebble.ValueStorageLowReadLatency
+		return policy, lockTableEndKey, nil
+	}
+
 	// Disable multi-level compaction heuristic for now. See #134423
 	// for why this was disabled, and what needs to be changed to reenable it.
 	// This issue tracks re-enablement: https://github.com/cockroachdb/pebble/issues/4139