Merge #148680

craig[bot] · stevendanna · craig[bot] · commit 39d519b21a48 · 2025-06-25T06:21:59.000Z
148680: kvserver,storage: allow sequence regression during lock flush r=arulajmani a=stevendanna Typically, when you acquire a replicated lock of the same or greater strength of an unreplicated lock, your unreplicated lock is freed from the lock table. However, this doesn't happen for all lock acquisition patterns. For instance, if the lock table is tracking a replicated lock for the key already, we do not free unreplicated locks for the new acquisition. However, even if we DID free such unreplicated locks, we may not want to in the future because we could be rolling back protection at a lower sequence number than the replicated lock. With our default settings, the above behaviour is fine. However, when we attempt to flush the unreplicated locks in the lock table to disk, we encounter an error: cannot acquire lock with strength Shared at seq number 0, already held at higher seq number 1 The comment around this error explains the two previous cases that justified this being an error. It assumed that any regression was either a replay or the result of a previous bug. Lock flushing produces a third case in which a regression of the sequence number is expected. Since sequence numbers are only used to track rolled back writes, it should be OK to allow it. But, since we only expect it in this single case, a new flag is added to MVCCAcquireLock for the caller to signal that they expect such a regression. Fixes #148674 Release note: None Co-authored-by: Steven Danna <danna@cockroachlabs.com>
diff --git a/pkg/kv/kvserver/batcheval/cmd_lease.go b/pkg/kv/kvserver/batcheval/cmd_lease.go
@@ -84,7 +84,7 @@ func evalNewLease(
 			} else {
 				for _, acq := range acquisitions {
 					if err := storage.MVCCAcquireLock(ctx, readWriter,
-						&acq.Txn, acq.IgnoredSeqNums, acq.Strength, acq.Key, ms, 0, 0); err != nil {
+						&acq.Txn, acq.IgnoredSeqNums, acq.Strength, acq.Key, ms, 0, 0, true /* allowSequenceNumberRegression */); err != nil {
 						return newFailedLeaseTrigger(isTransfer), err
 					}
 				}
diff --git a/pkg/kv/kvserver/batcheval/cmd_query_resolved_timestamp_test.go b/pkg/kv/kvserver/batcheval/cmd_query_resolved_timestamp_test.go
@@ -53,7 +53,7 @@ func TestQueryResolvedTimestamp(t *testing.T) {
 	}
 	writeLock := func(k string, str lock.Strength) {
 		txn := roachpb.MakeTransaction("test", roachpb.Key(k), 0, 0, makeTS(1), 0, 1, 0, false /* omitInRangefeeds */)
-		err := storage.MVCCAcquireLock(ctx, db, &txn.TxnMeta, txn.IgnoredSeqNums, str, roachpb.Key(k), nil, 0, 0)
+		err := storage.MVCCAcquireLock(ctx, db, &txn.TxnMeta, txn.IgnoredSeqNums, str, roachpb.Key(k), nil, 0, 0, false)
 		require.NoError(t, err)
 	}
 
diff --git a/pkg/kv/kvserver/batcheval/cmd_subsume.go b/pkg/kv/kvserver/batcheval/cmd_subsume.go
@@ -199,7 +199,7 @@ func Subsume(
 			statsDelta := enginepb.MVCCStats{}
 			for _, acq := range acquisitions {
 				if err := storage.MVCCAcquireLock(ctx, readWriter,
-					&acq.Txn, acq.IgnoredSeqNums, acq.Strength, acq.Key, &statsDelta, 0, 0); err != nil {
+					&acq.Txn, acq.IgnoredSeqNums, acq.Strength, acq.Key, &statsDelta, 0, 0, true /* allowSequenceNumberRegression */); err != nil {
 					return result.Result{}, err
 				}
 			}
diff --git a/pkg/kv/kvserver/batcheval/lock.go b/pkg/kv/kvserver/batcheval/lock.go
@@ -233,7 +233,7 @@ func acquireLockOnKey(
 		// conflicts with un-contended replicated locks -- we need to do so before
 		// we can acquire our own replicated lock; do that now, and also acquire
 		// the replicated lock if no conflicts are found.
-		if err := storage.MVCCAcquireLock(ctx, readWriter, &txn.TxnMeta, txn.IgnoredSeqNums, str, key, ms, maxLockConflicts, targetLockConflictBytes); err != nil {
+		if err := storage.MVCCAcquireLock(ctx, readWriter, &txn.TxnMeta, txn.IgnoredSeqNums, str, key, ms, maxLockConflicts, targetLockConflictBytes, false); err != nil {
 			return roachpb.LockAcquisition{}, err
 		}
 	default:
diff --git a/pkg/kv/kvserver/batcheval/lock_test.go b/pkg/kv/kvserver/batcheval/lock_test.go
@@ -195,9 +195,9 @@ func TestTxnBoundReplicatedLockTableView(t *testing.T) {
 	txn2 := roachpb.MakeTransaction("txn2", keyA, isolation.Serializable, roachpb.NormalUserPriority, makeTS(100, 0), 0, 0, 0, false)
 
 	// Have txn1 acquire 2 locks with different strengths.
-	err = storage.MVCCAcquireLock(ctx, engine, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Exclusive, keyA, nil, 0, 0)
+	err = storage.MVCCAcquireLock(ctx, engine, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Exclusive, keyA, nil, 0, 0, false)
 	require.NoError(t, err)
-	err = storage.MVCCAcquireLock(ctx, engine, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Shared, keyB, nil, 0, 0)
+	err = storage.MVCCAcquireLock(ctx, engine, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Shared, keyB, nil, 0, 0, false)
 	require.NoError(t, err)
 
 	reader := engine.NewReader(storage.StandardDurability)
diff --git a/pkg/kv/kvserver/gc/gc_test.go b/pkg/kv/kvserver/gc/gc_test.go
@@ -148,9 +148,9 @@ func TestLockAgeThresholdSetting(t *testing.T) {
 		require.NoError(t, err)
 		// Acquire some shared and exclusive locks as well.
 		for _, txn := range []*roachpb.Transaction{&txn1, &txn2} {
-			require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, makeKey(local, lock.Shared), nil, 0, 0))
+			require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, makeKey(local, lock.Shared), nil, 0, 0, false))
 		}
-		require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Exclusive, makeKey(local, lock.Exclusive), nil, 0, 0))
+		require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Exclusive, makeKey(local, lock.Exclusive), nil, 0, 0, false))
 	}
 	require.NoError(t, eng.Flush())
 
@@ -217,9 +217,9 @@ func TestIntentCleanupBatching(t *testing.T) {
 			idx := i*len(objectKeys) + j
 			switch idx % 3 {
 			case 0:
-				require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, key, nil, 0, 0))
+				require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, key, nil, 0, 0, false))
 			case 1:
-				require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Exclusive, key, nil, 0, 0))
+				require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Exclusive, key, nil, 0, 0, false))
 			case 2:
 				_, err := storage.MVCCPut(ctx, eng, key, intentHlc, value, storage.MVCCWriteOptions{Txn: &txn})
 				require.NoError(t, err)
diff --git a/pkg/kv/kvserver/rangefeed/task_test.go b/pkg/kv/kvserver/rangefeed/task_test.go
@@ -311,7 +311,7 @@ func TestInitResolvedTSScan(t *testing.T) {
 			roachpb.MakeLock(&txn1.TxnMeta, roachpb.Key("p"), lock.Exclusive),
 		}
 		for _, l := range testLocks {
-			err := storage.MVCCAcquireLock(ctx, engine, &txn1.TxnMeta, txn1.IgnoredSeqNums, l.Strength, l.Key, nil, 0, 0)
+			err := storage.MVCCAcquireLock(ctx, engine, &txn1.TxnMeta, txn1.IgnoredSeqNums, l.Strength, l.Key, nil, 0, 0, false)
 			require.NoError(t, err)
 		}
 		return engine
diff --git a/pkg/kv/kvserver/replica_consistency_test.go b/pkg/kv/kvserver/replica_consistency_test.go
@@ -328,7 +328,7 @@ func TestReplicaChecksumSHA512(t *testing.T) {
 	for i, l := range locks {
 		txnID := uuid.FromUint128(uint128.FromInts(0, uint64(l.txnID)))
 		txn := &roachpb.Transaction{TxnMeta: enginepb.TxnMeta{ID: txnID}}
-		require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, l.str, roachpb.Key(l.key), nil, 0, 0))
+		require.NoError(t, storage.MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, l.str, roachpb.Key(l.key), nil, 0, 0, false))
 
 		rd, err = CalcReplicaDigest(ctx, desc, eng, kvpb.ChecksumMode_CHECK_FULL, unlim, nil /* settings */)
 		require.NoError(t, err)
diff --git a/pkg/storage/bench_test.go b/pkg/storage/bench_test.go
@@ -1950,11 +1950,11 @@ func runMVCCAcquireLockCommon(
 				txn = &txn2
 			}
 			// Acquire a shared and an exclusive lock on the key.
-			err := MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, key, nil, 0, 0)
+			err := MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, key, nil, 0, 0, false)
 			if err != nil {
 				b.Fatal(err)
 			}
-			err = MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Exclusive, key, nil, 0, 0)
+			err = MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Exclusive, key, nil, 0, 0, false)
 			if err != nil {
 				b.Fatal(err)
 			}
@@ -1978,7 +1978,7 @@ func runMVCCAcquireLockCommon(
 		if checkFor {
 			err = MVCCCheckForAcquireLock(ctx, rw, txn, strength, key, 0, 0)
 		} else {
-			err = MVCCAcquireLock(ctx, rw, &txn.TxnMeta, txn.IgnoredSeqNums, strength, key, ms, 0, 0)
+			err = MVCCAcquireLock(ctx, rw, &txn.TxnMeta, txn.IgnoredSeqNums, strength, key, ms, 0, 0, false)
 		}
 		if heldOtherTxn {
 			if err == nil {
diff --git a/pkg/storage/engine_test.go b/pkg/storage/engine_test.go
@@ -1534,9 +1534,9 @@ func TestGetIntent(t *testing.T) {
 	// Key "b" has an intent, an exclusive lock, and a shared lock from txn1.
 	// NOTE: acquire in increasing strength order so that acquisition is never
 	// skipped.
-	err = MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Shared, keyB, nil, 0, 0)
+	err = MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Shared, keyB, nil, 0, 0, false)
 	require.NoError(t, err)
-	err = MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Exclusive, keyB, nil, 0, 0)
+	err = MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Exclusive, keyB, nil, 0, 0, false)
 	require.NoError(t, err)
 	_, err = MVCCPut(ctx, eng, keyB, txn1.ReadTimestamp, val, MVCCWriteOptions{Txn: txn1})
 	require.NoError(t, err)
@@ -1546,15 +1546,15 @@ func TestGetIntent(t *testing.T) {
 	require.NoError(t, err)
 
 	// Key "d" has an exclusive lock and a shared lock from txn2.
-	err = MVCCAcquireLock(ctx, eng, &txn2.TxnMeta, txn2.IgnoredSeqNums, lock.Shared, keyD, nil, 0, 0)
+	err = MVCCAcquireLock(ctx, eng, &txn2.TxnMeta, txn2.IgnoredSeqNums, lock.Shared, keyD, nil, 0, 0, false)
 	require.NoError(t, err)
-	err = MVCCAcquireLock(ctx, eng, &txn2.TxnMeta, txn2.IgnoredSeqNums, lock.Exclusive, keyD, nil, 0, 0)
+	err = MVCCAcquireLock(ctx, eng, &txn2.TxnMeta, txn2.IgnoredSeqNums, lock.Exclusive, keyD, nil, 0, 0, false)
 	require.NoError(t, err)
 
 	// Key "e" has a shared lock from each txn.
-	err = MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Shared, keyE, nil, 0, 0)
+	err = MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, lock.Shared, keyE, nil, 0, 0, false)
 	require.NoError(t, err)
-	err = MVCCAcquireLock(ctx, eng, &txn2.TxnMeta, txn2.IgnoredSeqNums, lock.Shared, keyE, nil, 0, 0)
+	err = MVCCAcquireLock(ctx, eng, &txn2.TxnMeta, txn2.IgnoredSeqNums, lock.Shared, keyE, nil, 0, 0, false)
 	require.NoError(t, err)
 
 	// Key "f" has no intent/locks.
@@ -1642,7 +1642,7 @@ func TestScanLocks(t *testing.T) {
 		if str == lock.Intent {
 			_, err = MVCCPut(ctx, eng, roachpb.Key(k), txn1.ReadTimestamp, roachpb.MakeValueFromBytes(roachpb.Key(k)), MVCCWriteOptions{Txn: txn1})
 		} else {
-			err = MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, str, roachpb.Key(k), nil, 0, 0)
+			err = MVCCAcquireLock(ctx, eng, &txn1.TxnMeta, txn1.IgnoredSeqNums, str, roachpb.Key(k), nil, 0, 0, false)
 		}
 		require.NoError(t, err)
 	}
@@ -2211,11 +2211,11 @@ func TestScanConflictingIntentsForDroppingLatchesEarly(t *testing.T) {
 			setup: func(t *testing.T, rw ReadWriter, _ *roachpb.Transaction) {
 				txnA := newTxn(belowTxnTS)
 				txnB := newTxn(belowTxnTS)
-				err := MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err := MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
-				err = MVCCAcquireLock(ctx, rw, &txnB.TxnMeta, txnB.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err = MVCCAcquireLock(ctx, rw, &txnB.TxnMeta, txnB.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
-				err = MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Exclusive, keyB, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err = MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Exclusive, keyB, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
 			},
 			start:                 keyA,
@@ -2229,9 +2229,9 @@ func TestScanConflictingIntentsForDroppingLatchesEarly(t *testing.T) {
 			name: "shared and exclusive locks should be ignored no end key",
 			setup: func(t *testing.T, rw ReadWriter, _ *roachpb.Transaction) {
 				txnA := newTxn(belowTxnTS)
-				err := MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err := MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
-				err = MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Exclusive, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err = MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Exclusive, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
 			},
 			start:                 keyA,
@@ -2244,11 +2244,11 @@ func TestScanConflictingIntentsForDroppingLatchesEarly(t *testing.T) {
 			setup: func(t *testing.T, rw ReadWriter, _ *roachpb.Transaction) {
 				txnA := newTxn(belowTxnTS)
 				txnB := newTxn(belowTxnTS)
-				err := MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err := MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
-				err = MVCCAcquireLock(ctx, rw, &txnB.TxnMeta, txnB.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err = MVCCAcquireLock(ctx, rw, &txnB.TxnMeta, txnB.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
-				err = MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Exclusive, keyB, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err = MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Exclusive, keyB, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
 				require.NoError(t, err)
 				_, err = MVCCPut(ctx, rw, keyC, txnA.WriteTimestamp, val, MVCCWriteOptions{Txn: txnA})
@@ -2264,11 +2264,11 @@ func TestScanConflictingIntentsForDroppingLatchesEarly(t *testing.T) {
 			setup: func(t *testing.T, rw ReadWriter, txn *roachpb.Transaction) {
 				txnA := newTxn(belowTxnTS)
 				txnB := newTxn(belowTxnTS)
-				err := MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err := MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
-				err = MVCCAcquireLock(ctx, rw, &txnB.TxnMeta, txnB.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err = MVCCAcquireLock(ctx, rw, &txnB.TxnMeta, txnB.IgnoredSeqNums, lock.Shared, keyA, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
-				err = MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Exclusive, keyB, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/)
+				err = MVCCAcquireLock(ctx, rw, &txnA.TxnMeta, txnA.IgnoredSeqNums, lock.Exclusive, keyB, nil /*ms*/, 0 /*maxConflicts*/, 0 /*targetLockConflictBytes*/, false)
 				require.NoError(t, err)
 				_, err = MVCCPut(ctx, rw, keyC, txn.WriteTimestamp, val, MVCCWriteOptions{Txn: txn})
 				require.NoError(t, err)
diff --git a/pkg/storage/metamorphic/operations.go b/pkg/storage/metamorphic/operations.go
@@ -290,7 +290,7 @@ func (m mvccAcquireLockOp) run(ctx context.Context) string {
 	txn := m.m.getTxn(m.txn)
 	writer := m.m.getReadWriter(m.writer)
 
-	err := storage.MVCCAcquireLock(ctx, writer, &txn.TxnMeta, txn.IgnoredSeqNums, m.strength, m.key, nil, int64(m.maxLockConflicts), m.targetLockConflictBytes)
+	err := storage.MVCCAcquireLock(ctx, writer, &txn.TxnMeta, txn.IgnoredSeqNums, m.strength, m.key, nil, int64(m.maxLockConflicts), m.targetLockConflictBytes, false)
 	if err != nil {
 		return fmt.Sprintf("error: %s", err)
 	}
diff --git a/pkg/storage/mvcc.go b/pkg/storage/mvcc.go
@@ -5974,6 +5974,7 @@ func MVCCAcquireLock(
 	ms *enginepb.MVCCStats,
 	maxLockConflicts int64,
 	targetLockConflictBytes int64,
+	allowSequenceNumberRegression bool,
 ) error {
 	if txn == nil {
 		// Non-transactional requests cannot acquire locks that outlive their
@@ -6027,7 +6028,8 @@ func MVCCAcquireLock(
 				continue
 			}
 			// If the found lock has the same strength as the acquisition then this is
-			// an unexpected case. We are likely part of a replayed batch and either:
+			// usually an unexpected case. We are likely part of a replayed batch and either:
+			//
 			// 1. the lock was reacquired at a later sequence number and the minimum
 			//    acquisition sequence number was not properly retained (bug!). See
 			//    below about why we preserve the earliest non-rolled back sequence
@@ -6036,10 +6038,21 @@ func MVCCAcquireLock(
 			//    subsequently acquired again at a higher sequence number. In such
 			//    cases, we can return an error as the client is no longer waiting for
 			//    a response.
-			return errors.Errorf(
-				"cannot acquire lock with strength %s at seq number %d, "+
-					"already held at higher seq number %d",
-				str.String(), txn.Sequence, foundLock.Txn.Sequence)
+			//
+			// IFF the caller sets allowSequenceNumberRegression, we don't consider
+			// this an error. The only callers who set this are callers who are
+			// flushing locks that were originally taken as unreplicated locks as
+			// replicated locks. In this case, it is possible that the unreplicated
+			// lock was being tracked at a lower sequence number and that the
+			// replicated re-acquisition didn't clear it from the lock table.
+			if !allowSequenceNumberRegression {
+				return errors.Errorf(
+					"cannot acquire lock with strength %s at seq number %d, "+
+						"already held at higher seq number %d in txn %s",
+					str.String(), txn.Sequence, foundLock.Txn.Sequence, txn.ID)
+			} else {
+				rolledBack = true
+			}
 		} else if enginepb.TxnSeqIsIgnored(foundLock.Txn.Sequence, ignoredSeqNums) {
 			// Acquiring at same epoch and new sequence number after
 			// previous sequence number was rolled back.
diff --git a/pkg/storage/mvcc_history_test.go b/pkg/storage/mvcc_history_test.go
@@ -1075,13 +1075,14 @@ func cmdAcquireLock(e *evalCtx) error {
 		str := e.getStrength()
 		maxLockConflicts := e.getMaxLockConflicts()
 		targetLockConflictBytes := e.getTargetLockConflictBytes()
+		allowSequenceNumberRegression := e.hasArg("allow_sequence_number_regression")
 		var txnMeta *enginepb.TxnMeta
 		var ignoredSeq []enginepb.IgnoredSeqNumRange
 		if txn != nil {
 			txnMeta = &txn.TxnMeta
 			ignoredSeq = txn.IgnoredSeqNums
 		}
-		return storage.MVCCAcquireLock(e.ctx, rw, txnMeta, ignoredSeq, str, key, e.ms, maxLockConflicts, targetLockConflictBytes)
+		return storage.MVCCAcquireLock(e.ctx, rw, txnMeta, ignoredSeq, str, key, e.ms, maxLockConflicts, targetLockConflictBytes, allowSequenceNumberRegression)
 	})
 }
 
diff --git a/pkg/storage/mvcc_stats_test.go b/pkg/storage/mvcc_stats_test.go
@@ -2049,7 +2049,7 @@ func TestMVCCStatsRandomized(t *testing.T) {
 			txnMeta = &s.Txn.TxnMeta
 			ignoredSeq = s.Txn.IgnoredSeqNums
 		}
-		if err := MVCCAcquireLock(ctx, s.batch, txnMeta, ignoredSeq, str, s.key, s.MSDelta, 0, 0); err != nil {
+		if err := MVCCAcquireLock(ctx, s.batch, txnMeta, ignoredSeq, str, s.key, s.MSDelta, 0, 0, false); err != nil {
 			return false, err.Error()
 		}
 		return true, ""
diff --git a/pkg/storage/mvcc_test.go b/pkg/storage/mvcc_test.go
@@ -2207,7 +2207,7 @@ func TestMVCCClearTimeRange(t *testing.T) {
 
 	// Add a shared lock at k1 with a txn at ts3.
 	addLock := func(t *testing.T, rw ReadWriter) {
-		err := MVCCAcquireLock(ctx, rw, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, testKey1, nil, 0, 0)
+		err := MVCCAcquireLock(ctx, rw, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, testKey1, nil, 0, 0, false)
 		require.NoError(t, err)
 	}
 	t.Run("clear everything hitting lock fails", func(t *testing.T) {
@@ -7573,6 +7573,7 @@ func TestApproximateLockTableSize(t *testing.T) {
 		stats,
 		0,
 		0,
+		false,
 	))
 	require.GreaterOrEqual(t, ApproximateLockTableSize(&acq), stats.LockBytes)
 }
diff --git a/pkg/storage/sst_test.go b/pkg/storage/sst_test.go
@@ -90,7 +90,7 @@ func TestCheckSSTConflictsMaxLockConflicts(t *testing.T) {
 		if i%2 != 0 {
 			str = lock.Exclusive
 		}
-		require.NoError(t, MVCCAcquireLock(ctx, batch, &txn1.TxnMeta, txn1.IgnoredSeqNums, str, roachpb.Key(key), nil, 0, 0))
+		require.NoError(t, MVCCAcquireLock(ctx, batch, &txn1.TxnMeta, txn1.IgnoredSeqNums, str, roachpb.Key(key), nil, 0, 0, false))
 	}
 	require.NoError(t, batch.Commit(true))
 	batch.Close()
diff --git a/pkg/storage/testdata/mvcc_histories/replicated_locks_errors b/pkg/storage/testdata/mvcc_histories/replicated_locks_errors
@@ -54,7 +54,16 @@ with t=A
 >> at end:
 txn: "A" meta={id=00000001 key=/Min iso=Serializable pri=0.00000000 epo=0 ts=10.000000000,0 min=0,0 seq=5} lock=true stat=PENDING rts=10.000000000,0 gul=0,0
 lock (Replicated): "k1"/Shared -> txn={id=00000001 key=/Min iso=Serializable pri=0.00000000 epo=0 ts=10.000000000,0 min=0,0 seq=10} ts=10.000000000,0 del=false klen=0 vlen=0 mergeTs=<nil> txnDidNotUpdateMeta=true
-error: (*withstack.withStack:) cannot acquire lock with strength Shared at seq number 5, already held at higher seq number 10
+error: (*withstack.withStack:) cannot acquire lock with strength Shared at seq number 5, already held at higher seq number 10 in txn 00000001-0000-0000-0000-000000000000
+
+run ok
+with t=A
+  txn_step seq=5
+  acquire_lock t=A k=k1 str=shared allow_sequence_number_regression
+----
+>> at end:
+txn: "A" meta={id=00000001 key=/Min iso=Serializable pri=0.00000000 epo=0 ts=10.000000000,0 min=0,0 seq=5} lock=true stat=PENDING rts=10.000000000,0 gul=0,0
+lock (Replicated): "k1"/Shared -> txn={id=00000001 key=/Min iso=Serializable pri=0.00000000 epo=0 ts=10.000000000,0 min=0,0 seq=5} ts=10.000000000,0 del=false klen=0 vlen=0 mergeTs=<nil> txnDidNotUpdateMeta=false
 
 run ok
 with t=A

Original file line number	Diff line number	Diff line change
`@@ -84,7 +84,7 @@ func evalNewLease(`
`84`	`84`	`} else {`
`85`	`85`	`for _, acq := range acquisitions {`
`86`	`86`	`if err := storage.MVCCAcquireLock(ctx, readWriter,`
`87`		`- &acq.Txn, acq.IgnoredSeqNums, acq.Strength, acq.Key, ms, 0, 0); err != nil {`
	`87`	`+ &acq.Txn, acq.IgnoredSeqNums, acq.Strength, acq.Key, ms, 0, 0, true /* allowSequenceNumberRegression */); err != nil {`
`88`	`88`	`return newFailedLeaseTrigger(isTransfer), err`
`89`	`89`	`}`
`90`	`90`	`}`
Original file line number	Diff line number	Diff line change
`@@ -53,7 +53,7 @@ func TestQueryResolvedTimestamp(t *testing.T) {`
`53`	`53`	`}`
`54`	`54`	`writeLock := func(k string, str lock.Strength) {`
`55`	`55`	`txn := roachpb.MakeTransaction("test", roachpb.Key(k), 0, 0, makeTS(1), 0, 1, 0, false /* omitInRangefeeds */)`
`56`		`- err := storage.MVCCAcquireLock(ctx, db, &txn.TxnMeta, txn.IgnoredSeqNums, str, roachpb.Key(k), nil, 0, 0)`
	`56`	`+ err := storage.MVCCAcquireLock(ctx, db, &txn.TxnMeta, txn.IgnoredSeqNums, str, roachpb.Key(k), nil, 0, 0, false)`
`57`	`57`	`require.NoError(t, err)`
`58`	`58`	`}`
`59`	`59`
Original file line number	Diff line number	Diff line change
`@@ -199,7 +199,7 @@ func Subsume(`
`199`	`199`	`statsDelta := enginepb.MVCCStats{}`
`200`	`200`	`for _, acq := range acquisitions {`
`201`	`201`	`if err := storage.MVCCAcquireLock(ctx, readWriter,`
`202`		`- &acq.Txn, acq.IgnoredSeqNums, acq.Strength, acq.Key, &statsDelta, 0, 0); err != nil {`
	`202`	`+ &acq.Txn, acq.IgnoredSeqNums, acq.Strength, acq.Key, &statsDelta, 0, 0, true /* allowSequenceNumberRegression */); err != nil {`
`203`	`203`	`return result.Result{}, err`
`204`	`204`	`}`
`205`	`205`	`}`
Original file line number	Diff line number	Diff line change
`@@ -233,7 +233,7 @@ func acquireLockOnKey(`
`233`	`233`	`// conflicts with un-contended replicated locks -- we need to do so before`
`234`	`234`	`// we can acquire our own replicated lock; do that now, and also acquire`
`235`	`235`	`// the replicated lock if no conflicts are found.`
`236`		`- if err := storage.MVCCAcquireLock(ctx, readWriter, &txn.TxnMeta, txn.IgnoredSeqNums, str, key, ms, maxLockConflicts, targetLockConflictBytes); err != nil {`
	`236`	`+ if err := storage.MVCCAcquireLock(ctx, readWriter, &txn.TxnMeta, txn.IgnoredSeqNums, str, key, ms, maxLockConflicts, targetLockConflictBytes, false); err != nil {`
`237`	`237`	`return roachpb.LockAcquisition{}, err`
`238`	`238`	`}`
`239`	`239`	`default:`
Original file line number	Diff line number	Diff line change
`@@ -311,7 +311,7 @@ func TestInitResolvedTSScan(t *testing.T) {`
`311`	`311`	`roachpb.MakeLock(&txn1.TxnMeta, roachpb.Key("p"), lock.Exclusive),`
`312`	`312`	`}`
`313`	`313`	`for _, l := range testLocks {`
`314`		`- err := storage.MVCCAcquireLock(ctx, engine, &txn1.TxnMeta, txn1.IgnoredSeqNums, l.Strength, l.Key, nil, 0, 0)`
	`314`	`+ err := storage.MVCCAcquireLock(ctx, engine, &txn1.TxnMeta, txn1.IgnoredSeqNums, l.Strength, l.Key, nil, 0, 0, false)`
`315`	`315`	`require.NoError(t, err)`
`316`	`316`	`}`
`317`	`317`	`return engine`
Original file line number	Diff line number	Diff line change
`@@ -1950,11 +1950,11 @@ func runMVCCAcquireLockCommon(`
`1950`	`1950`	`txn = &txn2`
`1951`	`1951`	`}`
`1952`	`1952`	`// Acquire a shared and an exclusive lock on the key.`
`1953`		`- err := MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, key, nil, 0, 0)`
	`1953`	`+ err := MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Shared, key, nil, 0, 0, false)`
`1954`	`1954`	`if err != nil {`
`1955`	`1955`	`b.Fatal(err)`
`1956`	`1956`	`}`
`1957`		`- err = MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Exclusive, key, nil, 0, 0)`
	`1957`	`+ err = MVCCAcquireLock(ctx, eng, &txn.TxnMeta, txn.IgnoredSeqNums, lock.Exclusive, key, nil, 0, 0, false)`
`1958`	`1958`	`if err != nil {`
`1959`	`1959`	`b.Fatal(err)`
`1960`	`1960`	`}`
`@@ -1978,7 +1978,7 @@ func runMVCCAcquireLockCommon(`
`1978`	`1978`	`if checkFor {`
`1979`	`1979`	`err = MVCCCheckForAcquireLock(ctx, rw, txn, strength, key, 0, 0)`
`1980`	`1980`	`} else {`
`1981`		`- err = MVCCAcquireLock(ctx, rw, &txn.TxnMeta, txn.IgnoredSeqNums, strength, key, ms, 0, 0)`
	`1981`	`+ err = MVCCAcquireLock(ctx, rw, &txn.TxnMeta, txn.IgnoredSeqNums, strength, key, ms, 0, 0, false)`
`1982`	`1982`	`}`
`1983`	`1983`	`if heldOtherTxn {`
`1984`	`1984`	`if err == nil {`