kvserver: add Store metrics for raft log size

Epic: none
Release note: none

Author: pav-kv

docs/generated/metrics/metrics.html

@@ -531,6 +531,8 @@
 <tr><td>STORAGE</td><td>raft.transport.sends-dropped</td><td>Number of Raft message sends dropped by the Raft Transport</td><td>Messages</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>raft.transport.sent</td><td>Number of Raft messages sent by the Raft Transport</td><td>Messages</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>raftlog.behind</td><td>Number of Raft log entries followers on other stores are behind.<br/><br/>This gauge provides a view of the aggregate number of log entries the Raft leaders<br/>on this node think the followers are behind. Since a raft leader may not always<br/>have a good estimate for this information for all of its followers, and since<br/>followers are expected to be behind (when they are not required as part of a<br/>quorum) *and* the aggregate thus scales like the count of such followers, it is<br/>difficult to meaningfully interpret this metric.</td><td>Log Entries</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
+<tr><td>STORAGE</td><td>raftlog.size.max</td><td>Approximate size of the largest Raft log on the store.</td><td>Bytes</td><td>GAUGE</td><td>BYTES</td><td>AVG</td><td>NONE</td></tr>
+<tr><td>STORAGE</td><td>raftlog.size.total</td><td>Approximate size of all Raft logs on the store.</td><td>Bytes</td><td>GAUGE</td><td>BYTES</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>raftlog.truncated</td><td>Number of Raft log entries truncated</td><td>Log Entries</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>range.adds</td><td>Number of range additions</td><td>Range Ops</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>range.merges</td><td>Number of range merges</td><td>Range Ops</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>

pkg/kv/kvserver/metrics.go

@@ -1799,6 +1799,19 @@ difficult to meaningfully interpret this metric.`,
 		Unit:        metric.Unit_COUNT,
 	}
 
+	metaRaftLogTotalSize = metric.Metadata{
+		Name:        "raftlog.size.total",
+		Help:        "Approximate size of all Raft logs on the store.",
+		Measurement: "Bytes",
+		Unit:        metric.Unit_BYTES,
+	}
+	metaRaftLogMaxSize = metric.Metadata{
+		Name:        "raftlog.size.max",
+		Help:        "Approximate size of the largest Raft log on the store.",
+		Measurement: "Bytes",
+		Unit:        metric.Unit_BYTES,
+	}
+
 	metaRaftFollowerPaused = metric.Metadata{
 		Name: "admission.raft.paused_replicas",
 		Help: `Number of followers (i.e. Replicas) to which replication is currently paused to help them recover from I/O overload.
@@ -2874,6 +2887,8 @@ type StoreMetrics struct {
 	// Raft log metrics.
 	RaftLogFollowerBehindCount *metric.Gauge
 	RaftLogTruncated           *metric.Counter
+	RaftLogTotalSize           *metric.Gauge
+	RaftLogMaxSize             *metric.Gauge
 
 	RaftPausedFollowerCount       *metric.Gauge
 	RaftPausedFollowerDroppedMsgs *metric.Counter
@@ -3643,6 +3658,8 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		// Raft log metrics.
 		RaftLogFollowerBehindCount: metric.NewGauge(metaRaftLogFollowerBehindCount),
 		RaftLogTruncated:           metric.NewCounter(metaRaftLogTruncated),
+		RaftLogTotalSize:           metric.NewGauge(metaRaftLogTotalSize),
+		RaftLogMaxSize:             metric.NewGauge(metaRaftLogMaxSize),
 
 		RaftPausedFollowerCount:       metric.NewGauge(metaRaftFollowerPaused),
 		RaftPausedFollowerDroppedMsgs: metric.NewCounter(metaRaftPausedFollowerDroppedMsgs),

pkg/kv/kvserver/replica_metrics.go

@@ -54,6 +54,7 @@ type ReplicaMetrics struct {
 	Underreplicated          bool
 	Overreplicated           bool
 	Decommissioning          bool
+	RaftLogSize              int64
 	RaftLogTooLarge          bool
 	RangeTooLarge            bool
 	BehindCount              int64
@@ -213,6 +214,7 @@ func calcReplicaMetrics(d calcReplicaMetricsInput) ReplicaMetrics {
 		Underreplicated:           underreplicated,
 		Overreplicated:            overreplicated,
 		Decommissioning:           decommissioning,
+		RaftLogSize:               d.raftLogSize,
 		RaftLogTooLarge: d.raftLogSizeTrusted &&
 			d.raftLogSize > raftLogTooLargeMultiple*d.raftCfg.RaftLogTruncationThreshold,
 		RangeTooLarge:            tooLarge,

pkg/kv/kvserver/replica_test.go

@@ -9275,6 +9275,7 @@ func TestReplicaMetrics(t *testing.T) {
 				Unavailable:         false,
 				Underreplicated:     false,
 				BehindCount:         10,
+				RaftLogSize:         5 * cfg.RaftLogTruncationThreshold,
 				RaftLogTooLarge:     true,
 				LeaderNotFortified:  true,
 				RaftFlowStateCounts: [3]int64{1, 0, 0},

pkg/kv/kvserver/store.go

@@ -3359,6 +3359,9 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error {
 		averageWriteBytesPerSecond     float64
 		averageCPUNanosPerSecond       float64
 
+		totalRaftLogSize int64
+		maxRaftLogSize   int64
+
 		rangeCount                int64
 		unavailableRangeCount     int64
 		underreplicatedRangeCount int64
@@ -3489,6 +3492,9 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error {
 		s.metrics.RecentReplicaCPUNanosPerSecond.RecordValue(replicaCPUNanosPerSecond)
 		s.metrics.RecentReplicaQueriesPerSecond.RecordValue(loadStats.QueriesPerSecond)
 
+		totalRaftLogSize += metrics.RaftLogSize
+		maxRaftLogSize = max(maxRaftLogSize, metrics.RaftLogSize)
+
 		locks += metrics.LockTableMetrics.Locks
 		totalLockHoldDurationNanos += metrics.LockTableMetrics.TotalLockHoldDurationNanos
 		locksWithWaitQueues += metrics.LockTableMetrics.LocksWithWaitQueues
@@ -3527,6 +3533,8 @@ func (s *Store) updateReplicationGauges(ctx context.Context) error {
 	for state, cnt := range raftFlowStateCounts {
 		s.metrics.RaftFlowStateCounts[state].Update(cnt)
 	}
+	s.metrics.RaftLogTotalSize.Update(totalRaftLogSize)
+	s.metrics.RaftLogMaxSize.Update(maxRaftLogSize)
 	s.metrics.AverageQueriesPerSecond.Update(averageQueriesPerSecond)
 	s.metrics.AverageRequestsPerSecond.Update(averageRequestsPerSecond)
 	s.metrics.AverageWritesPerSecond.Update(averageWritesPerSecond)