split: add an access direction metric and log

Similar to how the split decider logs and increments a counter when a
popular key is found, do the same for when the access direction is > 80%
left or right (descending or ascending).

Fixes: #142208
Epic: CRDB-43150

Release note (ops change): Adds a new metric `kv.loadsplitter.cleardirection`
which increments when replica samples are found to be moving
in a uniform direction.

Author: angles-n-daemons

docs/generated/metrics/metrics.html

@@ -202,6 +202,7 @@
 <tr><td>STORAGE</td><td>kv.concurrency.max_lock_hold_duration_nanos</td><td>Maximum length of time any lock in a lock table is held. Does not include replicated locks (intents) that are not held in memory</td><td>Nanoseconds</td><td>GAUGE</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>kv.concurrency.max_lock_wait_duration_nanos</td><td>Maximum lock wait duration across requests currently waiting in lock wait-queues</td><td>Nanoseconds</td><td>GAUGE</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>kv.concurrency.max_lock_wait_queue_waiters_for_lock</td><td>Maximum number of requests actively waiting in any single lock wait-queue</td><td>Lock-Queue Waiters</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
+<tr><td>STORAGE</td><td>kv.loadsplitter.cleardirection</td><td>Load-based splitter observed an access direction greater than 80% left or right in the samples.</td><td>Occurrences</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>kv.loadsplitter.nosplitkey</td><td>Load-based splitter could not find a split key.</td><td>Occurrences</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>kv.loadsplitter.popularkey</td><td>Load-based splitter could not find a split key and the most popular sampled split key occurs in &gt;= 25% of the samples.</td><td>Occurrences</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>kv.prober.planning_attempts</td><td>Number of attempts at planning out probes made; in order to probe KV we need to plan out which ranges to probe;</td><td>Runs</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>

pkg/kv/kvserver/asim/state/split_decider.go

@@ -80,8 +80,9 @@ func NewSplitDecider(settings *config.SimulationSettings) *SplitDecider {
 func (s *SplitDecider) newDecider() *split.Decider {
 	decider := &split.Decider{}
 	split.Init(decider, s.splitConfig, &split.LoadSplitterMetrics{
-		PopularKeyCount: metric.NewCounter(metric.Metadata{}),
-		NoSplitKeyCount: metric.NewCounter(metric.Metadata{}),
+		PopularKeyCount:     metric.NewCounter(metric.Metadata{}),
+		NoSplitKeyCount:     metric.NewCounter(metric.Metadata{}),
+		ClearDirectionCount: metric.NewCounter(metric.Metadata{}),
 	}, split.SplitQPS)
 	return decider
 }

pkg/kv/kvserver/metrics.go

@@ -2471,6 +2471,13 @@ Note that the measurement does not include the duration for replicating the eval
 		Unit:        metric.Unit_COUNT,
 	}
 
+	metaClearDirectionCount = metric.Metadata{
+		Name:        "kv.loadsplitter.cleardirection",
+		Help:        "Load-based splitter observed an access direction greater than 80% left or right in the samples.",
+		Measurement: "Occurrences",
+		Unit:        metric.Unit_COUNT,
+	}
+
 	metaSplitEstimatedStats = metric.Metadata{
 		Name:        "kv.split.estimated_stats",
 		Help:        "Number of splits that computed estimated MVCC stats.",
@@ -3333,8 +3340,9 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		registry:              storeRegistry,
 		TenantsStorageMetrics: newTenantsStorageMetrics(),
 		LoadSplitterMetrics: &split.LoadSplitterMetrics{
-			PopularKeyCount: metric.NewCounter(metaPopularKeyCount),
-			NoSplitKeyCount: metric.NewCounter(metaNoSplitKeyCount),
+			PopularKeyCount:     metric.NewCounter(metaPopularKeyCount),
+			NoSplitKeyCount:     metric.NewCounter(metaNoSplitKeyCount),
+			ClearDirectionCount: metric.NewCounter(metaClearDirectionCount),
 		},
 
 		// Replica metrics.

pkg/kv/kvserver/split/decider.go

@@ -9,6 +9,7 @@ package split
 
 import (
 	"context"
+	"math"
 	"math/rand/v2"
 	"time"
 
@@ -140,8 +141,9 @@ func GlobalRandSource() RandSource {
 
 // LoadSplitterMetrics consists of metrics for load-based splitter split key.
 type LoadSplitterMetrics struct {
-	PopularKeyCount *metric.Counter
-	NoSplitKeyCount *metric.Counter
+	PopularKeyCount     *metric.Counter
+	NoSplitKeyCount     *metric.Counter
+	ClearDirectionCount *metric.Counter
 }
 
 // Decider tracks the latest load and if certain conditions are met, records
@@ -283,6 +285,13 @@ func (d *Decider) recordLocked(
 						if popularKeyFrequency >= splitKeyThreshold {
 							d.loadSplitterMetrics.PopularKeyCount.Inc(1)
 						}
+						accessDirection := d.mu.splitFinder.AccessDirection()
+						direction := directionStr(accessDirection)
+						log.KvDistribution.Infof(ctx, "%s, access balance between left and right for sampled keys: %s-biased %d%%",
+							causeMsg, direction, int(math.Abs(accessDirection)*100))
+						if math.Abs(accessDirection) >= clearDirectionThreshold {
+							d.loadSplitterMetrics.ClearDirectionCount.Inc(1)
+						}
 						d.loadSplitterMetrics.NoSplitKeyCount.Inc(1)
 					}
 				}
@@ -542,3 +551,16 @@ func (t *maxStatTracker) windowWidth() time.Duration {
 	// NB: -1 because during a rotation, only len(t.windows)-1 windows survive.
 	return t.minRetention / time.Duration(len(t.windows)-1)
 }
+
+// Returns the absolute percentage and direction of accesses
+// as a string to be used in a log statement.
+func directionStr(direction float64) string {
+	dstr := "right"
+	if direction == 0 {
+		dstr = "even"
+	}
+	if direction < 0 {
+		dstr = "left"
+	}
+	return dstr
+}

pkg/kv/kvserver/split/decider_test.go

@@ -64,6 +64,14 @@ func ms(i int) time.Time {
 	return ts.Add(time.Duration(i) * time.Millisecond)
 }
 
+func newSplitterMetrics() *LoadSplitterMetrics {
+	return &LoadSplitterMetrics{
+		PopularKeyCount:     metric.NewCounter(metric.Metadata{}),
+		NoSplitKeyCount:     metric.NewCounter(metric.Metadata{}),
+		ClearDirectionCount: metric.NewCounter(metric.Metadata{}),
+	}
+}
+
 func TestDecider(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 
@@ -76,10 +84,7 @@ func TestDecider(t *testing.T) {
 	}
 
 	var d Decider
-	Init(&d, &loadSplitConfig, &LoadSplitterMetrics{
-		PopularKeyCount: metric.NewCounter(metric.Metadata{}),
-		NoSplitKeyCount: metric.NewCounter(metric.Metadata{}),
-	},
+	Init(&d, &loadSplitConfig, newSplitterMetrics(),
 		SplitQPS,
 	)
 
@@ -242,10 +247,7 @@ func TestDecider_MaxStat(t *testing.T) {
 	}
 
 	var d Decider
-	Init(&d, &loadSplitConfig, &LoadSplitterMetrics{
-		PopularKeyCount: metric.NewCounter(metric.Metadata{}),
-		NoSplitKeyCount: metric.NewCounter(metric.Metadata{}),
-	}, SplitQPS)
+	Init(&d, &loadSplitConfig, newSplitterMetrics(), SplitQPS)
 
 	assertMaxStat := func(i int, expMaxStat float64, expOK bool) {
 		t.Helper()
@@ -408,10 +410,7 @@ func TestSplitStatisticsGeneral(t *testing.T) {
 				statThreshold: 1,
 			}
 
-			Init(&decider, &loadSplitConfig, &LoadSplitterMetrics{
-				PopularKeyCount: metric.NewCounter(metric.Metadata{}),
-				NoSplitKeyCount: metric.NewCounter(metric.Metadata{}),
-			}, SplitCPU)
+			Init(&decider, &loadSplitConfig, newSplitterMetrics(), SplitCPU)
 
 			for i := 1; i <= 1000; i++ {
 				k := i
@@ -462,10 +461,7 @@ func TestSplitStatisticsPopularKey(t *testing.T) {
 				statThreshold: 1,
 			}
 
-			Init(&decider, &loadSplitConfig, &LoadSplitterMetrics{
-				PopularKeyCount: metric.NewCounter(metric.Metadata{}),
-				NoSplitKeyCount: metric.NewCounter(metric.Metadata{}),
-			}, SplitCPU)
+			Init(&decider, &loadSplitConfig, newSplitterMetrics(), SplitCPU)
 
 			for i := 1; i <= 1000; i++ {
 				decider.Record(context.Background(), ms(timeStart+i*50), ld(1), func() roachpb.Span {
@@ -491,12 +487,9 @@ func TestDeciderMetrics(t *testing.T) {
 		statThreshold: 1,
 	}
 
-	Init(&dPopular, &loadSplitConfig, &LoadSplitterMetrics{
-		PopularKeyCount: metric.NewCounter(metric.Metadata{}),
-		NoSplitKeyCount: metric.NewCounter(metric.Metadata{}),
-	}, SplitCPU)
+	Init(&dPopular, &loadSplitConfig, newSplitterMetrics(), SplitCPU)
 
-	// No split key, popular key
+	// No split key, popular key, clear direction
 	for i := 0; i < 20; i++ {
 		dPopular.Record(context.Background(), ms(timeStart), ld(1), func() roachpb.Span {
 			return roachpb.Span{Key: keys.SystemSQLCodec.TablePrefix(uint32(0))}
@@ -510,13 +503,11 @@ func TestDeciderMetrics(t *testing.T) {
 
 	assert.Equal(t, dPopular.loadSplitterMetrics.PopularKeyCount.Count(), int64(2))
 	assert.Equal(t, dPopular.loadSplitterMetrics.NoSplitKeyCount.Count(), int64(2))
+	assert.Equal(t, dPopular.loadSplitterMetrics.ClearDirectionCount.Count(), int64(2))
 
-	// No split key, not popular key
+	// No split key, not popular key, clear direction
 	var dNotPopular Decider
-	Init(&dNotPopular, &loadSplitConfig, &LoadSplitterMetrics{
-		PopularKeyCount: metric.NewCounter(metric.Metadata{}),
-		NoSplitKeyCount: metric.NewCounter(metric.Metadata{}),
-	}, SplitCPU)
+	Init(&dNotPopular, &loadSplitConfig, newSplitterMetrics(), SplitCPU)
 
 	for i := 0; i < 20; i++ {
 		dNotPopular.Record(context.Background(), ms(timeStart), ld(1), func() roachpb.Span {
@@ -531,13 +522,29 @@ func TestDeciderMetrics(t *testing.T) {
 
 	assert.Equal(t, dNotPopular.loadSplitterMetrics.PopularKeyCount.Count(), int64(0))
 	assert.Equal(t, dNotPopular.loadSplitterMetrics.NoSplitKeyCount.Count(), int64(2))
+	assert.Equal(t, dNotPopular.loadSplitterMetrics.ClearDirectionCount.Count(), int64(2))
+
+	// no split key, no popular key, no clear direction
+	var dNoClearDirection Decider
+	Init(&dNoClearDirection, &loadSplitConfig, newSplitterMetrics(), SplitCPU)
+	for i := 0; i < 20; i++ {
+		dNoClearDirection.Record(context.Background(), ms(timeStart), ld(1), func() roachpb.Span {
+			return roachpb.Span{Key: keys.SystemSQLCodec.TablePrefix(uint32(i))}
+		})
+	}
+	for i := 1; i <= 2000; i++ {
+		dNoClearDirection.Record(context.Background(), ms(timeStart+i*1000), ld(1), func() roachpb.Span {
+			return roachpb.Span{Key: keys.SystemSQLCodec.TablePrefix(uint32(i % 20))}
+		})
+	}
+
+	assert.Equal(t, dNoClearDirection.loadSplitterMetrics.PopularKeyCount.Count(), int64(0))
+	assert.Equal(t, dNoClearDirection.loadSplitterMetrics.NoSplitKeyCount.Count(), int64(0))
+	assert.Equal(t, dNoClearDirection.loadSplitterMetrics.ClearDirectionCount.Count(), int64(0))
 
 	// No split key, all insufficient counters
 	var dAllInsufficientCounters Decider
-	Init(&dAllInsufficientCounters, &loadSplitConfig, &LoadSplitterMetrics{
-		PopularKeyCount: metric.NewCounter(metric.Metadata{}),
-		NoSplitKeyCount: metric.NewCounter(metric.Metadata{}),
-	}, SplitCPU)
+	Init(&dAllInsufficientCounters, &loadSplitConfig, newSplitterMetrics(), SplitCPU)
 	for i := 0; i < 20; i++ {
 		dAllInsufficientCounters.Record(context.Background(), ms(timeStart), ld(1), func() roachpb.Span {
 			return roachpb.Span{Key: keys.SystemSQLCodec.TablePrefix(uint32(0))}
@@ -551,4 +558,6 @@ func TestDeciderMetrics(t *testing.T) {
 
 	assert.Equal(t, dAllInsufficientCounters.loadSplitterMetrics.PopularKeyCount.Count(), int64(0))
 	assert.Equal(t, dAllInsufficientCounters.loadSplitterMetrics.NoSplitKeyCount.Count(), int64(0))
+	assert.Equal(t, dAllInsufficientCounters.loadSplitterMetrics.ClearDirectionCount.Count(), int64(0))
+
 }

pkg/kv/kvserver/split/load_based_splitter_test.go

@@ -22,7 +22,6 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
 	"github.com/cockroachdb/cockroach/pkg/testutils/skip"
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
-	"github.com/cockroachdb/cockroach/pkg/util/metric"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/cockroach/pkg/workload/ycsb"
 	"github.com/cockroachdb/datadriven"
@@ -325,10 +324,7 @@ func (dc deciderConfig) makeDecider(randSource rand.Source) *Decider {
 		statThreshold: dc.threshold,
 	}
 
-	Init(d, &loadSplitConfig, &LoadSplitterMetrics{
-		PopularKeyCount: metric.NewCounter(metric.Metadata{}),
-		NoSplitKeyCount: metric.NewCounter(metric.Metadata{}),
-	}, dc.objective)
+	Init(d, &loadSplitConfig, newSplitterMetrics(), dc.objective)
 	return d
 }
 

pkg/kv/kvserver/split/unweighted_finder.go

@@ -47,6 +47,7 @@ const (
 	splitKeyMinCounter         = 100              // min aggregate counters before consideration
 	splitKeyThreshold          = 0.25             // 25% difference between left/right counters
 	splitKeyContainedThreshold = 0.50             // too many spanning queries over split point
+	clearDirectionThreshold    = 0.80             // 80% accesses left or right
 )
 
 type sample struct {