Merge #147267

147267: kvserver: plan for replica creation and destruction r=tbg a=pav-kv

Epic: CRDB-49111

Co-authored-by: Pavel Kalinnikov <[email protected]>

Author: craig[bot]

pkg/kv/kvserver/kvstorage/destroy.go

@@ -91,6 +91,22 @@ func ClearRangeData(
 	return nil
 }
 
+// DestroyReplicaTODO is the plan for splitting DestroyReplica into cross-engine
+// writes.
+//
+//  1. Log storage write (durable):
+//     1.1. Write WAG node with the state machine mutation (2).
+//  2. State machine mutation:
+//     2.1. Clear RangeID-local un-/replicated state.
+//     2.2. (optional) Clear replicated MVCC span.
+//     2.3. Write RangeTombstone with next ReplicaID/LogID.
+//  3. Log engine GC (after state machine mutation 2 is durably applied):
+//     3.1. Remove previous LogID.
+//
+// TODO(sep-raft-log): support the status quo in which 1+2+3 is written
+// atomically, and 1.1 is not written.
+const DestroyReplicaTODO = 0
+
 // DestroyReplica destroys all or a part of the Replica's state, installing a
 // RangeTombstone in its place. Due to merges, splits, etc, there is a need
 // to control which part of the state this method actually gets to remove,
@@ -114,6 +130,7 @@ func DestroyReplica(
 	if diskReplicaID.ReplicaID >= nextReplicaID {
 		return errors.AssertionFailedf("replica r%d/%d must not survive its own tombstone", rangeID, diskReplicaID)
 	}
+	_ = DestroyReplicaTODO // 2.1 + 3.1 + 2.2
 	if err := ClearRangeData(ctx, rangeID, reader, writer, opts); err != nil {
 		return err
 	}
@@ -129,6 +146,7 @@ func DestroyReplica(
 		return errors.AssertionFailedf(
 			"cannot rewind tombstone from %d to %d", ts.NextReplicaID, nextReplicaID)
 	}
+	_ = DestroyReplicaTODO // 2.3
 	return sl.SetRangeTombstone(ctx, writer, kvserverpb.RangeTombstone{
 		NextReplicaID: nextReplicaID, // NB: nextReplicaID > 0
 	})

pkg/kv/kvserver/kvstorage/replica_state.go

@@ -101,6 +101,17 @@ func (r LoadedReplicaState) check(storeID roachpb.StoreID) error {
 	return nil
 }
 
+// CreateUninitReplicaTODO is the plan for splitting CreateUninitializedReplica
+// into cross-engine writes.
+//
+//  1. Log storage write (durable):
+//     1.1. Write WAG node with the state machine mutation (2).
+//  2. State machine mutation:
+//     2.1. Write RaftReplicaID with the new ReplicaID/LogID.
+//
+// TODO(sep-raft-log): support the status quo in which only 2.1 is written.
+const CreateUninitReplicaTODO = 0
+
 // CreateUninitializedReplica creates an uninitialized replica in storage.
 // Returns kvpb.RaftGroupDeletedError if this replica can not be created
 // because it has been deleted.
@@ -134,6 +145,7 @@ func CreateUninitializedReplica(
 	//   the Term and Vote values for that older replica in the context of
 	//   this newer replica is harmless since it just limits the votes for
 	//   this replica.
+	_ = CreateUninitReplicaTODO
 	if err := sl.SetRaftReplicaID(ctx, eng, replicaID); err != nil {
 		return err
 	}

pkg/kv/kvserver/replica_raftstorage.go

@@ -414,6 +414,28 @@ func (r *Replica) updateRangeInfo(ctx context.Context, desc *roachpb.RangeDescri
 	return nil
 }
 
+// applySnapshotTODO is the plan for splitting snapshot application into
+// cross-engine writes.
+//
+//  1. Log engine write (durable):
+//     1.1. HardState, RaftTruncatedState for new LogID. Log is empty.
+//     1.2. WAG node with the state machine mutation (2).
+//
+//  2. State machine mutation:
+//     2.1. For subsumed, clear RangeID-local un-/replicated state.
+//     2.2. For subsumed, write RangeTombstone with max NextReplicaID / LogID.
+//     2.3. Clear MVCC keyspace for (this + subsumed).
+//     2.4. Ingest snapshot SSTs.
+//     2.5. Update RaftReplicaID with the new LogID.
+//
+//  3. Log engine GC (after state machine mutation 2 is durably applied):
+//     3.1. Remove previous LogID.
+//     3.2. For each subsumed, remove the last LogID.
+//
+// TODO(sep-raft-log): support the status quo in which 1+2+3 is written
+// atomically, and 1.2 is not written.
+const applySnapshotTODO = 0
+
 // applySnapshotRaftMuLocked updates the replica and its store based on the given
 // (non-empty) snapshot and associated HardState. All snapshots must pass
 // through Raft for correctness, i.e. the parameters to this method must be
@@ -577,6 +599,7 @@ func (r *Replica) applySnapshotRaftMuLocked(
 		subsumedDescs: subsumedDescs,
 	}
 
+	_ = applySnapshotTODO
 	clearedUnreplicatedSpan, clearedSubsumedSpans, err := prepareSnapApply(ctx, prepInput)
 	if err != nil {
 		return err
@@ -611,12 +634,14 @@ func (r *Replica) applySnapshotRaftMuLocked(
 	var ingestStats pebble.IngestOperationStats
 	var writeBytes uint64
 	if applyAsIngest {
+		_ = applySnapshotTODO // all atomic
 		exciseSpan := desc.KeySpan().AsRawSpanWithNoLocals()
 		if ingestStats, err = r.store.TODOEngine().IngestAndExciseFiles(ctx, inSnap.SSTStorageScratch.SSTs(), inSnap.sharedSSTs, inSnap.externalSSTs, exciseSpan); err != nil {
 			return errors.Wrapf(err, "while ingesting %s and excising %s-%s",
 				inSnap.SSTStorageScratch.SSTs(), exciseSpan.Key, exciseSpan.EndKey)
 		}
 	} else {
+		_ = applySnapshotTODO // all atomic
 		err := r.store.TODOEngine().ConvertFilesToBatchAndCommit(
 			ctx, inSnap.SSTStorageScratch.SSTs(), clearedSpans)
 		if err != nil {

pkg/kv/kvserver/snapshot_apply_prepare.go

@@ -44,17 +44,19 @@ func prepareSnapApply(
 	[]roachpb.Span, // clearedSubsumedSpans
 	error,
 ) {
-	// Step 1: Write unreplicated SST
+	_ = applySnapshotTODO // 3.1 + 1.1 + 2.5.
 	unreplicatedSSTFile, clearedUnreplicatedSpan, err := writeUnreplicatedSST(
 		ctx, input.id, input.st, input.truncState, input.hardState, input.sl,
 	)
 	if err != nil {
 		return roachpb.Span{}, nil, err
 	}
+	_ = applySnapshotTODO // add to 2.4.
 	if err := input.writeSST(ctx, unreplicatedSSTFile.Data()); err != nil {
 		return roachpb.Span{}, nil, err
 	}
 
+	_ = applySnapshotTODO // 3.2 + 2.1 + 2.2 + 2.3
 	clearedSubsumedSpans, err := clearSubsumedReplicaDiskData(
 		ctx, input.st, input.todoEng, input.writeSST,
 		input.desc, input.subsumedDescs,

pkg/kv/kvserver/store_create_replica.go

@@ -203,6 +203,7 @@ func (s *Store) tryGetOrCreateReplica(
 	// be accessed by someone holding a reference to, or currently creating a
 	// Replica for this rangeID, and that's us.
 
+	_ = kvstorage.CreateUninitReplicaTODO
 	if err := kvstorage.CreateUninitializedReplica(
 		// TODO(sep-raft-log): needs both engines due to tombstone (which lives on
 		// statemachine).