kvserver: wait for lease upgrade

This fixes the race between a lease upgrade request and
BumpSideTransportClosed, which fails if there is a pending proposal.

Epic: none
Release note: none

Author: pav-kv

pkg/kv/kvserver/replica_closedts_test.go

@@ -694,6 +694,7 @@ func TestRejectedLeaseDoesntDictateClosedTimestamp(t *testing.T) {
 // we eliminate some memory copying and pass the node liveness record in to the
 // function so that we only have to grab it once instead of on each call to
 // BumpSideTransportClosed, we should be able to reach our target latency.
+// TODO(pav-kv): in 2025, this shows about 180-200ns.
 func BenchmarkBumpSideTransportClosed(b *testing.B) {
 	defer leaktest.AfterTest(b)()
 	defer log.Scope(b).Close(b)
@@ -721,6 +722,22 @@ func BenchmarkBumpSideTransportClosed(b *testing.B) {
 	now := s.Clock().NowAsClockTimestamp()
 	targets := map[ctpb.RangeClosedTimestampPolicy]hlc.Timestamp{}
 
+	// Wait for the scratch range's lease upgrade from expiration-based lease. The
+	// closed timestamp bumping requires for there to be no in-flight proposals,
+	// so a racing lease upgrade can cause a test failure like in #151295.
+	//
+	// TODO(pav-kv): share this code with TestCluster.MaybeWaitForLeaseUpgrade().
+	testutils.SucceedsSoon(b, func() error {
+		lease, _, err := s.GetRangeLease(ctx, key, roachpb.QueryLocalNodeOnly)
+		require.NoError(b, err)
+		typ := lease.Current().Type()
+		if typ == roachpb.LeaseExpiration {
+			return errors.Errorf("still on expiration-based lease")
+		}
+		b.Logf("lease is now of type: %s", typ)
+		return nil
+	})
+
 	b.ResetTimer()
 	for i := 0; i < b.N; i++ {
 		// Advance time and the closed timestamp target.