kvserver: add kv.closed_timestamp.policy_latency_info_missing

When a replica refreshes its policies, it looks up its peer replicas latency
info via a map passed by PolicyRefresher, which in turn periodically pulls node
latency info from RPCContext. If latency data for a node is missing, a default
hardcoded max RTT of 150ms is used.

Previously, it was hard to tell when this is happening. This commit adds metrics
to track how often the closed timestamp policy refresh falls back to the default
RTT due to missing node latency info. A high count might indicate the latency
cache isn’t refreshed frequently enough, suggesting we should consider lowering
kv.closed_timestamp.policy_latency_refresh_interval.

Resolves: #143890
Release note: none

Author: wenyihu6

docs/generated/metrics/metrics.html

@@ -212,6 +212,7 @@
 <tr><td>STORAGE</td><td>kv.closed_timestamp.policy.lead_for_global_reads_latency_less_than_80ms</td><td>Number of ranges with LEAD_FOR_GLOBAL_READS_LATENCY_LESS_THAN_80MS closed timestamp policy</td><td>Ranges</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>kv.closed_timestamp.policy.lead_for_global_reads_with_no_latency_info</td><td>Number of ranges with LEAD_FOR_GLOBAL_READS_WITH_NO_LATENCY_INFO closed timestamp policy</td><td>Ranges</td><td>GAUGE</td><td>COUNT</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>kv.closed_timestamp.policy_change</td><td>Number of times closed timestamp policy change occurred on ranges</td><td>Events</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
+<tr><td>STORAGE</td><td>kv.closed_timestamp.policy_latency_info_missing</td><td>Number of times closed timestamp policy refresh had to use hardcoded network RTT due to missing node latency info for one or more replicas</td><td>Events</td><td>COUNTER</td><td>COUNT</td><td>AVG</td><td>NON_NEGATIVE_DERIVATIVE</td></tr>
 <tr><td>STORAGE</td><td>kv.concurrency.avg_lock_hold_duration_nanos</td><td>Average lock hold duration across locks currently held in lock tables. Does not include replicated locks (intents) that are not held in memory</td><td>Nanoseconds</td><td>GAUGE</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>kv.concurrency.avg_lock_wait_duration_nanos</td><td>Average lock wait duration across requests currently waiting in lock wait-queues</td><td>Nanoseconds</td><td>GAUGE</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>
 <tr><td>STORAGE</td><td>kv.concurrency.latch_conflict_wait_durations</td><td>Durations in nanoseconds spent on latch acquisition waiting for conflicts with other latches</td><td>Nanoseconds</td><td>HISTOGRAM</td><td>NANOSECONDS</td><td>AVG</td><td>NONE</td></tr>

pkg/kv/kvserver/metrics.go

@@ -2442,6 +2442,14 @@ throttled they do count towards 'delay.total' and 'delay.enginebackpressure'.
 		Unit:        metric.Unit_COUNT,
 	}
 
+	metaClosedTimestampLatencyInfoMissing = metric.Metadata{
+		Name: "kv.closed_timestamp.policy_latency_info_missing",
+		Help: "Number of times closed timestamp policy refresh had to use hardcoded network RTT " +
+			"due to missing node latency info for one or more replicas",
+		Measurement: "Events",
+		Unit:        metric.Unit_COUNT,
+	}
+
 	// Replica circuit breaker.
 	metaReplicaCircuitBreakerCurTripped = metric.Metadata{
 		Name: "kv.replica_circuit_breaker.num_tripped_replicas",
@@ -3045,7 +3053,8 @@ type StoreMetrics struct {
 	ClosedTimestampMaxBehindNanos *metric.Gauge
 
 	// Closed timestamp policy change on ranges metrics.
-	ClosedTimestampPolicyChange *metric.Counter
+	ClosedTimestampPolicyChange       *metric.Counter
+	ClosedTimestampLatencyInfoMissing *metric.Counter
 
 	// Replica circuit breaker.
 	ReplicaCircuitBreakerCurTripped *metric.Gauge
@@ -3865,7 +3874,8 @@ func newStoreMetrics(histogramWindow time.Duration) *StoreMetrics {
 		SplitsWithEstimatedStats:     metric.NewCounter(metaSplitEstimatedStats),
 		SplitEstimatedTotalBytesDiff: metric.NewCounter(metaSplitEstimatedTotalBytesDiff),
 
-		ClosedTimestampPolicyChange: metric.NewCounter(metaClosedTimestampPolicyChange),
+		ClosedTimestampPolicyChange:       metric.NewCounter(metaClosedTimestampPolicyChange),
+		ClosedTimestampLatencyInfoMissing: metric.NewCounter(metaClosedTimestampLatencyInfoMissing),
 	}
 	sm.categoryIterMetrics.init(storeRegistry)
 

pkg/kv/kvserver/replica.go

@@ -1430,13 +1430,19 @@ func (r *Replica) RefreshPolicy(latencies map[roachpb.NodeID]time.Duration) {
 		// policy bucket. This then controls how far in the future timestamps will
 		// be closed for the range.
 		maxLatency := time.Duration(-1)
+		replicaLatencyInfoMissing := false
 		for _, peer := range desc.InternalReplicas {
 			peerLatency := closedts.DefaultMaxNetworkRTT
 			if latency, ok := latencies[peer.NodeID]; ok {
 				peerLatency = latency
+			} else {
+				replicaLatencyInfoMissing = true
 			}
 			maxLatency = max(maxLatency, peerLatency)
 		}
+		if replicaLatencyInfoMissing {
+			r.store.metrics.ClosedTimestampLatencyInfoMissing.Inc(1)
+		}
 		return closedts.FindBucketBasedOnNetworkRTTWithDampening(
 			oldPolicy,
 			maxLatency,