storage: aggressively separate values in range ID keyspace

Adapt Cockroach's implementation of pebble.SpanPolicyFunc to mark the range ID
keyspace with ValueStorageLatencyTolerant. This setting will cause Pebble to
separate values into blob files as long as they're large enough that a pointer
to an external value is likely to be smaller than the value itself. Separating
values improves write amplification by avoiding rewriting values during some
compactions.

We target the range ID keyspace because it contains the raft log, which is
written during all user writes but rarely read.

Tangentially related to #16624.

Epic: none
Release note (performance improvement): Reduces write amplification by storing
raft log values in separate blob files. This reduces write bandwidth,
especially on stores with many replicas. This in turn can increase throughput
and reduce latency. This behavior is active as long as the
storage.value_separation.enabled cluster setting is enabled.

Author: jbowens

pkg/storage/pebble.go

@@ -549,28 +549,7 @@ func DefaultPebbleOptions() *pebble.Options {
 	opts.TargetByteDeletionRate = 128 << 20 // 128 MB
 	opts.Experimental.ShortAttributeExtractor = shortAttributeExtractorForValues
 
-	lockTableStartKey := EncodeMVCCKey(MVCCKey{Key: keys.LocalRangeLockTablePrefix})
-	lockTableEndKey := EncodeMVCCKey(MVCCKey{Key: keys.LocalRangeLockTablePrefix.PrefixEnd()})
-	localEndKey := EncodeMVCCKey(MVCCKey{Key: keys.LocalPrefix.PrefixEnd()})
-	opts.Experimental.SpanPolicyFunc = func(startKey []byte) (policy pebble.SpanPolicy, endKey []byte, _ error) {
-		if !bytes.HasPrefix(startKey, keys.LocalPrefix) {
-			return pebble.SpanPolicy{}, nil, nil
-		}
-		// Prefer fast compression for all local keys, since they shouldn't take up
-		// a significant part of the space.
-		policy.PreferFastCompression = true
-
-		// We also disable value separation for lock keys.
-		if cockroachkvs.Compare(startKey, lockTableEndKey) >= 0 {
-			return policy, localEndKey, nil
-		}
-		if cockroachkvs.Compare(startKey, lockTableStartKey) < 0 {
-			return policy, lockTableStartKey, nil
-		}
-		policy.DisableValueSeparationBySuffix = true
-		policy.ValueStoragePolicy = pebble.ValueStorageLowReadLatency
-		return policy, lockTableEndKey, nil
-	}
+	opts.Experimental.SpanPolicyFunc = spanPolicyFunc
 	opts.Experimental.UserKeyCategories = userKeyCategories
 
 	opts.Levels[0] = pebble.LevelOptions{
@@ -607,6 +586,55 @@ func DefaultPebbleOptions() *pebble.Options {
 	return opts
 }
 
+var (
+	spanPolicyLocalRangeIDEndKey = EncodeMVCCKey(MVCCKey{Key: keys.LocalRangeIDPrefix.AsRawKey().PrefixEnd()})
+	spanPolicyLockTableStartKey  = EncodeMVCCKey(MVCCKey{Key: keys.LocalRangeLockTablePrefix})
+	spanPolicyLockTableEndKey    = EncodeMVCCKey(MVCCKey{Key: keys.LocalRangeLockTablePrefix.PrefixEnd()})
+	spanPolicyLocalEndKey        = EncodeMVCCKey(MVCCKey{Key: keys.LocalPrefix.PrefixEnd()})
+)
+
+// spanPolicyFunc is a pebble.SpanPolicyFunc that applies special policies for
+// the CockroachDB keyspace.
+func spanPolicyFunc(startKey []byte) (policy pebble.SpanPolicy, endKey []byte, _ error) {
+	// There's no special policy for non-local keys.
+	if !bytes.HasPrefix(startKey, keys.LocalPrefix) {
+		return pebble.SpanPolicy{}, nil, nil
+	}
+	// Prefer fast compression for all local keys, since they shouldn't take up
+	// a significant part of the space.
+	policy.PreferFastCompression = true
+
+	// The first section of the local keyspace is the Range-ID keyspace. It
+	// extends from the beginning of the keyspace to the Range Local keys. The
+	// Range-ID keyspace includes the raft log, which is rarely read and
+	// receives ~half the writes.
+	if cockroachkvs.Compare(startKey, spanPolicyLocalRangeIDEndKey) < 0 {
+		if !bytes.HasPrefix(startKey, keys.LocalRangeIDPrefix) {
+			return pebble.SpanPolicy{}, nil, errors.AssertionFailedf("startKey %s is not a Range-ID key", startKey)
+		}
+		policy.ValueStoragePolicy = pebble.ValueStorageLatencyTolerant
+		return policy, spanPolicyLocalRangeIDEndKey, nil
+	}
+
+	// We also disable value separation for lock keys.
+	if cockroachkvs.Compare(startKey, spanPolicyLockTableEndKey) >= 0 {
+		// Not a lock key, so use default value separation within sstable (by
+		// suffix) and into blob files.
+		// NB: there won't actually be a suffix in these local keys.
+		return policy, spanPolicyLocalEndKey, nil
+	}
+	if cockroachkvs.Compare(startKey, spanPolicyLockTableStartKey) < 0 {
+		// Not a lock key, so use default value separation within sstable (by
+		// suffix) and into blob files.
+		// NB: there won't actually be a suffix in these local keys.
+		return policy, spanPolicyLockTableStartKey, nil
+	}
+	// Lock key. Disable value separation.
+	policy.DisableValueSeparationBySuffix = true
+	policy.ValueStoragePolicy = pebble.ValueStorageLowReadLatency
+	return policy, spanPolicyLockTableEndKey, nil
+}
+
 func shortAttributeExtractorForValues(
 	key []byte, keyPrefixLen int, value []byte,
 ) (pebble.ShortAttribute, error) {

pkg/storage/pebble_test.go

@@ -1762,3 +1762,66 @@ func TestPebbleCompactCancellation(t *testing.T) {
 	bfs.WaitForBlockAndUnblock()
 	wg.Wait()
 }
+
+func TestPebbleSpanPolicyFunc(t *testing.T) {
+	defer leaktest.AfterTest(t)()
+	defer log.Scope(t).Close(t)
+
+	type testCase struct {
+		startKey   roachpb.Key
+		wantPolicy pebble.SpanPolicy
+		wantEndKey []byte
+	}
+	cases := []testCase{
+		{
+			startKey: keys.RaftHardStateKey(1),
+			wantPolicy: pebble.SpanPolicy{
+				PreferFastCompression: true,
+				ValueStoragePolicy:    pebble.ValueStorageLatencyTolerant,
+			},
+			wantEndKey: spanPolicyLocalRangeIDEndKey,
+		},
+		{
+			startKey: keys.RaftLogKey(9, 2),
+			wantPolicy: pebble.SpanPolicy{
+				PreferFastCompression: true,
+				ValueStoragePolicy:    pebble.ValueStorageLatencyTolerant,
+			},
+			wantEndKey: spanPolicyLocalRangeIDEndKey,
+		},
+		{
+			startKey: keys.RangeDescriptorKey(roachpb.RKey("a")),
+			wantPolicy: pebble.SpanPolicy{
+				PreferFastCompression: true,
+			},
+			wantEndKey: spanPolicyLockTableStartKey,
+		},
+		{
+			startKey: func() roachpb.Key {
+				k, _ := keys.LockTableSingleKey(roachpb.Key("a"), nil)
+				return k
+			}(),
+			wantPolicy: pebble.SpanPolicy{
+				PreferFastCompression:          true,
+				DisableValueSeparationBySuffix: true,
+				ValueStoragePolicy:             pebble.ValueStorageLowReadLatency,
+			},
+			wantEndKey: spanPolicyLockTableEndKey,
+		},
+		{
+			startKey:   keys.SystemSQLCodec.IndexPrefix(1, 2),
+			wantPolicy: pebble.SpanPolicy{},
+			wantEndKey: nil,
+		},
+	}
+
+	for _, tc := range cases {
+		t.Run(fmt.Sprintf("%x", tc.startKey), func(t *testing.T) {
+			ek := EngineKey{Key: tc.startKey}.Encode()
+			policy, endKey, err := spanPolicyFunc(ek)
+			require.NoError(t, err)
+			require.Equal(t, tc.wantPolicy, policy)
+			require.Equal(t, tc.wantEndKey, endKey)
+		})
+	}
+}