kvserver: deflake TestReplicaCircuitBreaker_Partial_Retry

The test was relying on global metric counters to assert
the behavior of a single request. This was error prone,
as we do not control what all other requests may be doing.

Instead, just grep the trace.

Closes #156284.

Author: tbg

pkg/kv/kvclient/kvcoord/dist_sender.go

@@ -370,6 +370,9 @@ const (
 	// The maximum number of times a replica is retried when it repeatedly returns
 	// stale lease info.
 	sameReplicaRetryLimit = 10
+	// InLeaseTransferBackoffTraceMessage is traced when DistSender backs off as a
+	// result of a NotLeaseholderError. It is exported for testing.
+	InLeaseTransferBackoffTraceMessage = "backing off due to NotLeaseHolderErr with stale info"
 )
 
 var rangeDescriptorCacheSize = settings.RegisterIntSetting(
@@ -3135,8 +3138,8 @@ func (ds *DistSender) sendToReplicas(
 					if shouldBackoff {
 						ds.metrics.InLeaseTransferBackoffs.Inc(1)
 						log.VErrEventf(ctx, 2,
-							"backing off due to NotLeaseHolderErr with stale info "+
-								"(updatedLH=%t intentionallySentToFollower=%t leaseholderUnavailable=%t)",
+							InLeaseTransferBackoffTraceMessage+
+								" (updatedLH=%t intentionallySentToFollower=%t leaseholderUnavailable=%t)",
 							updatedLeaseholder, intentionallySentToFollower, leaseholderUnavailable)
 					} else {
 						inTransferRetry.Reset() // The following Next() call will not block.

pkg/kv/kvserver/client_replica_circuit_breaker_test.go

@@ -863,7 +863,6 @@ func TestReplicaCircuitBreaker_Partial_Retry(t *testing.T) {
 			requireRUEs := func(t *testing.T, dbs []*kv.DB) {
 				t.Helper()
 				for _, db := range dbs {
-					backoffMetric := (db.NonTransactionalSender().(*kv.CrossRangeTxnWrapperSender)).Wrapped().(*kvcoord.DistSender).Metrics().InLeaseTransferBackoffs
 					ctx, finishAndGetRec := tracing.ContextWithRecordingSpan(context.Background(), db.Tracer, "requireRUEs")
 					defer func() {
 						rec := finishAndGetRec()
@@ -872,13 +871,12 @@ func TestReplicaCircuitBreaker_Partial_Retry(t *testing.T) {
 						}
 						t.Logf("failure trace:\n%s", rec)
 					}()
-					initialBackoff := backoffMetric.Count()
 					err := db.Put(ctx, key, value)
-					// Verify that we did not perform any backoff while executing this request.
-					require.EqualValues(t, 0, backoffMetric.Count()-initialBackoff)
 					require.Error(t, err)
 					require.True(t, errors.HasType(err, (*kvpb.ReplicaUnavailableError)(nil)),
 						"expected ReplicaUnavailableError, got %v", err)
+					// Verify that we did not perform any backoff while executing this request.
+					require.NotContains(t, finishAndGetRec(), kvcoord.InLeaseTransferBackoffTraceMessage)
 				}
 				t.Logf("writes failed with ReplicaUnavailableError")
 			}