kvserver: add directives to make replica destruction more interesting

Here, we add directives to add some dummy Raft log entries and state
machine keys to the replica on n1/s1 before it is destroyed.

Epic: none

Release note: None

Author: arulajmani

pkg/kv/kvserver/replica_lifecycle_datadriven_test.go

@@ -12,14 +12,19 @@ import (
 	"cmp"
 	"context"
 	"fmt"
+	"strconv"
 	"strings"
 	"testing"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/keys"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvpb"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/abortspan"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/batcheval"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/concurrency/lock"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvserverpb"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/kvstorage"
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/logstore"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/print"
 	"github.com/cockroachdb/cockroach/pkg/raft/raftpb"
 	"github.com/cockroachdb/cockroach/pkg/roachpb"
@@ -31,6 +36,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/leaktest"
 	"github.com/cockroachdb/cockroach/pkg/util/log"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
+	"github.com/cockroachdb/cockroach/pkg/util/uuid"
 	"github.com/cockroachdb/datadriven"
 	"github.com/cockroachdb/errors"
 	"github.com/stretchr/testify/require"
@@ -87,6 +93,20 @@ import (
 //	Destroys the replica on n1 for the specified range. The replica's state
 //	must have already been created via create-replica.
 //
+// append-raft-entries range-id=<int> num-entries=<int>
+// ----
+//
+//	Appends the specified number of dummy raft entries to the raft log for
+//	the replica on n1/s1. The replica must have already been created via
+//	create-replica.
+//
+// create-range-data range-id=<int> [num-user-keys=<int>] [num-system-keys=<int>] [num-lock-table-keys=<int>]
+// ----
+//
+//	Creates the specified number of user, system, and lock table keys for the
+//	range. At least one parameter should be non-zero to ensure this directive is
+//	not nonsensical.
+//
 // print-range-state [sort-keys=<bool>]
 // ----
 //
@@ -97,6 +117,8 @@ func TestReplicaLifecycleDataDriven(t *testing.T) {
 	defer leaktest.AfterTest(t)()
 	defer log.Scope(t).Close(t)
 
+	storage.DisableMetamorphicSimpleValueEncoding(t) // for deterministic output
+
 	datadriven.Walk(t, "testdata/replica_lifecycle", func(t *testing.T, path string) {
 		tc := newTestCtx()
 		defer tc.close()
@@ -303,7 +325,7 @@ func TestReplicaLifecycleDataDriven(t *testing.T) {
 					kvstorage.TODOReadWriter(batch),
 					kvstorage.DestroyReplicaInfo{
 						FullReplicaID: rs.replica.FullReplicaID,
-						Keys:          roachpb.RSpan{Key: rs.desc.StartKey, EndKey: rs.desc.EndKey},
+						Keys:          rs.desc.RSpan(),
 					},
 					rs.desc.NextReplicaID,
 				)
@@ -317,6 +339,80 @@ func TestReplicaLifecycleDataDriven(t *testing.T) {
 				rs.replica = nil
 				return output
 
+			case "append-raft-entries":
+				rangeID := dd.ScanArg[roachpb.RangeID](t, d, "range-id")
+				numEntries := dd.ScanArg[int](t, d, "num-entries")
+				rs := tc.mustGetRangeState(t, rangeID)
+				require.NotNil(t, rs.replica, "replica must be created before appending entries")
+
+				batch := tc.storage.NewBatch()
+				defer batch.Close()
+
+				sl := logstore.NewStateLoader(rangeID)
+				lastIndex := rs.replica.lastIdx
+				rs.replica.lastIdx += kvpb.RaftIndex(numEntries)
+				term := rs.replica.hs.Term
+
+				for i := 0; i < numEntries; i++ {
+					entryIndex := lastIndex + 1 + kvpb.RaftIndex(i)
+					require.NoError(t, storage.MVCCBlindPutProto(
+						ctx, batch,
+						sl.RaftLogKey(entryIndex), hlc.Timestamp{},
+						&raftpb.Entry{Index: uint64(entryIndex), Term: term},
+						storage.MVCCWriteOptions{},
+					))
+				}
+
+				output, err := print.DecodeWriteBatch(batch.Repr())
+				require.NoError(t, err)
+				err = batch.Commit(true)
+				require.NoError(t, err)
+				return strings.ReplaceAll(output, "\n\n", "\n")
+
+			case "create-range-data":
+				rangeID := dd.ScanArg[roachpb.RangeID](t, d, "range-id")
+				numUserKeys := dd.ScanArgOr(t, d, "num-user-keys", 0)
+				numSystemKeys := dd.ScanArgOr(t, d, "num-system-keys", 0)
+				numLockTableKeys := dd.ScanArgOr(t, d, "num-lock-table-keys", 0)
+				require.True(t, numUserKeys > 0 || numSystemKeys > 0 || numLockTableKeys > 0)
+
+				rs := tc.mustGetRangeState(t, rangeID)
+				batch := tc.storage.NewBatch()
+				defer batch.Close()
+
+				ts := hlc.Timestamp{WallTime: 1}
+
+				getUserKey := func(i int) roachpb.Key {
+					return append(rs.desc.StartKey.AsRawKey(), strconv.Itoa(i)...)
+				}
+
+				// 1. User keys.
+				for i := 0; i < numUserKeys; i++ {
+					require.NoError(t, batch.PutMVCC(
+						storage.MVCCKey{Key: getUserKey(i), Timestamp: ts}, storage.MVCCValue{},
+					))
+				}
+				// 2. System keys.
+				for i := 0; i < numSystemKeys; i++ {
+					key := keys.TransactionKey(getUserKey(i), uuid.NamespaceDNS)
+					require.NoError(t, batch.PutMVCC(
+						storage.MVCCKey{Key: key, Timestamp: ts}, storage.MVCCValue{},
+					))
+				}
+				// 3. Lock table keys.
+				for i := 0; i < numLockTableKeys; i++ {
+					ek, _ := storage.LockTableKey{
+						Key: getUserKey(i), Strength: lock.Intent, TxnUUID: uuid.UUID{},
+					}.ToEngineKey(nil)
+					require.NoError(t, batch.PutEngineKey(ek, nil))
+				}
+
+				output, err := print.DecodeWriteBatch(batch.Repr())
+				require.NoError(t, err)
+				err = batch.Commit(true)
+				require.NoError(t, err)
+				return output
+
 			case "print-range-state":
 				var sb strings.Builder
 				if len(tc.ranges) == 0 {
@@ -363,8 +459,9 @@ type rangeState struct {
 // engine (both raft log and state machine) state.
 type replicaInfo struct {
 	roachpb.FullReplicaID
-	hs raftpb.HardState
-	ts kvserverpb.RaftTruncatedState
+	hs      raftpb.HardState
+	ts      kvserverpb.RaftTruncatedState
+	lastIdx kvpb.RaftIndex
 }
 
 // testCtx is a single test's context. It tracks the state of all ranges and any
@@ -442,8 +539,9 @@ func (tc *testCtx) updatePostReplicaCreateState(
 			RangeID:   rs.desc.RangeID,
 			ReplicaID: replID.ReplicaID,
 		},
-		hs: hs,
-		ts: ts,
+		hs:      hs,
+		ts:      ts,
+		lastIdx: ts.Index,
 	}
 }
 
@@ -504,8 +602,9 @@ func (r *replicaInfo) String() string {
 	if r.hs == (raftpb.HardState{}) {
 		sb.WriteString("uninitialized")
 	} else {
-		sb.WriteString(fmt.Sprintf("HardState={Term:%d,Vote:%d,Commit:%d} ", r.hs.Term, r.hs.Vote, r.hs.Commit))
-		sb.WriteString(fmt.Sprintf("TruncatedState={Index:%d,Term:%d}", r.ts.Index, r.ts.Term))
+		sb.WriteString(fmt.Sprintf("HardState={Term:%d,Vote:%d,Commit:%d}", r.hs.Term, r.hs.Vote, r.hs.Commit))
+		sb.WriteString(fmt.Sprintf(" TruncatedState={Index:%d,Term:%d}", r.ts.Index, r.ts.Term))
+		sb.WriteString(fmt.Sprintf(" LastIdx=%d", r.lastIdx))
 	}
 	return sb.String()
 }

pkg/kv/kvserver/testdata/replica_lifecycle/create_replica.txt

@@ -29,4 +29,4 @@ print-range-state
 range desc: r1:{a-d} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4, gen=0]
 		replica (n1/s1): id=1 uninitialized
 range desc: r2:{d-k} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4, gen=0]
-		replica (n1/s1): id=1 HardState={Term:5,Vote:0,Commit:10} TruncatedState={Index:10,Term:5}
+		replica (n1/s1): id=1 HardState={Term:5,Vote:0,Commit:10} TruncatedState={Index:10,Term:5} LastIdx=10

pkg/kv/kvserver/testdata/replica_lifecycle/destroy_replica.txt

@@ -1,6 +1,6 @@
-create-descriptor start=a end=z replicas=(1,2,3)
+create-descriptor start=a end=c replicas=(1,2,3)
 ----
-created descriptor: r1:{a-z} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4, gen=0]
+created descriptor: r1:{a-c} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4, gen=0]
 
 create-replica range-id=1 initialized
 ----
@@ -25,3 +25,61 @@ Put: 0,0 /Local/RangeID/1/u/RangeTombstone (0x016989757266746200): next_replica_
 Delete (Sized at 38): 0,0 /Local/RangeID/1/u/RaftHardState (0x016989757266746800): 
 Delete: 0,0 /Local/RangeID/1/u/RaftReplicaID (0x016989757266747200): 
 Delete (Sized at 32): 0,0 /Local/RangeID/1/u/RaftTruncatedState (0x016989757266747400): 
+
+create-descriptor start=c end=f replicas=(1,2,3,4,5)
+----
+created descriptor: r2:{c-f} [(n1,s1):1, (n2,s2):2, (n3,s3):3, (n4,s4):4, (n5,s5):5, next=6, gen=0]
+
+create-replica range-id=2 initialized
+----
+created replica: (n1,s1):1
+Put: 0,0 /Local/RangeID/2/r/RangeLease (0x01698a72726c6c2d00): <empty>
+Put: 0,0 /Local/RangeID/2/r/RangeGCThreshold (0x01698a726c67632d00): 0,0
+Put: 0,0 /Local/RangeID/2/r/RangeGCHint (0x01698a727267636800): latest_range_delete_timestamp:<> gc_timestamp:<> gc_timestamp_next:<> 
+Put: 0,0 /Local/RangeID/2/r/RangeVersion (0x01698a727276657200): 10.8-upgrading-step-007
+Put: 0,0 /Local/RangeID/2/r/RangeAppliedState (0x01698a727261736b00): raft_applied_index:10 lease_applied_index:10 range_stats:<sys_bytes:142 sys_count:4 > raft_closed_timestamp:<> raft_applied_index_term:5 
+Put: 0,0 /Local/RangeID/2/u/RaftReplicaID (0x01698a757266747200): replica_id:1 
+Put: 0,0 /Local/RangeID/2/u/RaftHardState (0x01698a757266746800): term:5 vote:0 commit:10 lead:0 lead_epoch:0 
+Put: 0,0 /Local/RangeID/2/u/RaftTruncatedState (0x01698a757266747400): index:10 term:5 
+
+append-raft-entries range-id=2 num-entries=3
+----
+Put: 0,0 /Local/RangeID/2/u/RaftLog/logIndex:11 (0x01698a757266746c000000000000000b00): Term:5 Index:11 Type:EntryNormal : EMPTY
+Put: 0,0 /Local/RangeID/2/u/RaftLog/logIndex:12 (0x01698a757266746c000000000000000c00): Term:5 Index:12 Type:EntryNormal : EMPTY
+Put: 0,0 /Local/RangeID/2/u/RaftLog/logIndex:13 (0x01698a757266746c000000000000000d00): Term:5 Index:13 Type:EntryNormal : EMPTY
+
+create-range-data range-id=2 num-user-keys=2 num-system-keys=3 num-lock-table-keys=4
+----
+Put: 0.000000001,0 "c0" (0x633000000000000000000109): ""
+Put: 0.000000001,0 "c1" (0x633100000000000000000109): ""
+Put: 0.000000001,0 /Local/Range"c0"/Transaction/"6ba7b810-9dad-11d1-80b4-00c04fd430c8" (0x016b126330000174786e2d6ba7b8109dad11d180b400c04fd430c800000000000000000109): ""
+Put: 0.000000001,0 /Local/Range"c1"/Transaction/"6ba7b810-9dad-11d1-80b4-00c04fd430c8" (0x016b126331000174786e2d6ba7b8109dad11d180b400c04fd430c800000000000000000109): ""
+Put: 0.000000001,0 /Local/Range"c2"/Transaction/"6ba7b810-9dad-11d1-80b4-00c04fd430c8" (0x016b126332000174786e2d6ba7b8109dad11d180b400c04fd430c800000000000000000109): ""
+Put: "c0"/Intent/00000000-0000-0000-0000-000000000000: 
+Put: "c1"/Intent/00000000-0000-0000-0000-000000000000: 
+Put: "c2"/Intent/00000000-0000-0000-0000-000000000000: 
+Put: "c3"/Intent/00000000-0000-0000-0000-000000000000: 
+
+destroy-replica range-id=2
+----
+Delete (Sized at 28): 0,0 /Local/RangeID/2/r/RangeGCThreshold (0x01698a726c67632d00): 
+Delete (Sized at 43): 0,0 /Local/RangeID/2/r/RangeAppliedState (0x01698a727261736b00): 
+Delete (Sized at 34): 0,0 /Local/RangeID/2/r/RangeGCHint (0x01698a727267636800): 
+Delete (Sized at 44): 0,0 /Local/RangeID/2/r/RangeLease (0x01698a72726c6c2d00): 
+Delete (Sized at 36): 0,0 /Local/RangeID/2/r/RangeVersion (0x01698a727276657200): 
+Put: 0,0 /Local/RangeID/2/u/RangeTombstone (0x01698a757266746200): next_replica_id:6 
+Delete (Sized at 38): 0,0 /Local/RangeID/2/u/RaftHardState (0x01698a757266746800): 
+Delete (Sized at 42): 0,0 /Local/RangeID/2/u/RaftLog/logIndex:11 (0x01698a757266746c000000000000000b00): 
+Delete (Sized at 42): 0,0 /Local/RangeID/2/u/RaftLog/logIndex:12 (0x01698a757266746c000000000000000c00): 
+Delete (Sized at 42): 0,0 /Local/RangeID/2/u/RaftLog/logIndex:13 (0x01698a757266746c000000000000000d00): 
+Delete: 0,0 /Local/RangeID/2/u/RaftReplicaID (0x01698a757266747200): 
+Delete (Sized at 32): 0,0 /Local/RangeID/2/u/RaftTruncatedState (0x01698a757266747400): 
+Delete (Sized at 37): 0.000000001,0 /Local/Range"c0"/Transaction/"6ba7b810-9dad-11d1-80b4-00c04fd430c8" (0x016b126330000174786e2d6ba7b8109dad11d180b400c04fd430c800000000000000000109): 
+Delete (Sized at 37): 0.000000001,0 /Local/Range"c1"/Transaction/"6ba7b810-9dad-11d1-80b4-00c04fd430c8" (0x016b126331000174786e2d6ba7b8109dad11d180b400c04fd430c800000000000000000109): 
+Delete (Sized at 37): 0.000000001,0 /Local/Range"c2"/Transaction/"6ba7b810-9dad-11d1-80b4-00c04fd430c8" (0x016b126332000174786e2d6ba7b8109dad11d180b400c04fd430c800000000000000000109): 
+Delete (Sized at 27): /Local/Lock"c0" 0300000000000000000000000000000000 (0x017a6b126330000100030000000000000000000000000000000012): 
+Delete (Sized at 27): /Local/Lock"c1" 0300000000000000000000000000000000 (0x017a6b126331000100030000000000000000000000000000000012): 
+Delete (Sized at 27): /Local/Lock"c2" 0300000000000000000000000000000000 (0x017a6b126332000100030000000000000000000000000000000012): 
+Delete (Sized at 27): /Local/Lock"c3" 0300000000000000000000000000000000 (0x017a6b126333000100030000000000000000000000000000000012): 
+Delete (Sized at 12): 0.000000001,0 "c0" (0x633000000000000000000109): 
+Delete (Sized at 12): 0.000000001,0 "c1" (0x633100000000000000000109): 

pkg/kv/kvserver/testdata/replica_lifecycle/split_trigger.txt

@@ -39,7 +39,7 @@ Put: 0,0 /Local/RangeID/2/r/RangeAppliedState (0x01698a727261736b00): raft_appli
 print-range-state
 ----
 range desc: r1:{a-m} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4, gen=0]
-		replica (n1/s1): id=1 HardState={Term:5,Vote:0,Commit:10} TruncatedState={Index:10,Term:5}
+		replica (n1/s1): id=1 HardState={Term:5,Vote:0,Commit:10} TruncatedState={Index:10,Term:5} LastIdx=10
 		lease: repl=(n2,s2):2 seq=0 start=0,0 type=LeaseLeader term=10 min-exp=0.000000100,0 pro=0,0 acq=Unspecified
 range desc: r2:{m-z} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4, gen=0]
 		lease: repl=(n2,s2):2 seq=0 start=0,0 type=LeaseExpiration exp=0.000000109,0 pro=0,0 acq=Unspecified
@@ -88,7 +88,7 @@ Put: 0,0 /Local/RangeID/4/r/RangeAppliedState (0x01698c727261736b00): raft_appli
 print-range-state sort-keys=true
 ----
 range desc: r1:{a-f} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4, gen=0]
-		replica (n1/s1): id=1 HardState={Term:5,Vote:0,Commit:10} TruncatedState={Index:10,Term:5}
+		replica (n1/s1): id=1 HardState={Term:5,Vote:0,Commit:10} TruncatedState={Index:10,Term:5} LastIdx=10
 		lease: repl=(n1,s1):1 seq=0 start=0,0 type=LeaseEpoch epo=20 min-exp=0,0 pro=0,0 acq=Unspecified
 range desc: r3:{f-m} [(n1,s1):1, (n2,s2):2, (n3,s3):3, next=4, gen=0]
 		lease: repl=(n1,s1):1 seq=0 start=0,0 type=LeaseEpoch epo=20 min-exp=0,0 pro=0,0 acq=Unspecified