kvserver: start meta1 and meta2 as two separate ranges at bootstrap

Previously, we used to start meta1 and meta2 in one range at bootstrap,
and rely on load-based splitting to split them if needed. However, in
some cases, load-based splitting doesn't work when it decides to split a
point in meta1 (meta1 is not allowed to split).

This commit changes that by starting meta1 and meta2 as two separate
ranges at bootstrap. This way, if meta2 range needs to split more, it
can split because it won't collide with meta1.

In order to ensure that we get the descriptors correct, I started a
cluster (when meta1 and meta2 were on the same range), and manually
split them and captured the descriptors:

```
[1] Meta Key: /Meta1/""
    Range r1:
      StartKey: /Min
      EndKey:   /Meta2/""
      Replicas: (n1,s1):1
      Generation: 1

[2] Meta Key: /Meta1/Max
    Range r79:
      StartKey: /Meta2/""
      EndKey:   /System/NodeLiveness
      Replicas: (n1,s1):1
      Generation: 1

[3] Meta Key: /Meta2/System/NodeLiveness
    Range r79:
      StartKey: /Meta2/""
      EndKey:   /System/NodeLiveness
      Replicas: (n1,s1):1
      Generation: 1

[4] Meta Key: /Meta2/System/NodeLivenessMax
    Range r2:
      StartKey: /System/NodeLiveness
      EndKey:   /System/NodeLivenessMax
      Replicas: (n1,s1):1
      Generation: 0
```

After this commit, the first few ranges are bootstrapped with
the following descriptors:

```
[1] Meta Key: /Meta1/""
    Range r1:
      StartKey: /Min
      EndKey:   /Meta2/""
      Replicas: (n1,s1):1
      Generation: 0

[2] Meta Key: /Meta1/Max
    Range r2:
      StartKey: /Meta2/""
      EndKey:   /System/NodeLiveness
      Replicas: (n1,s1):1
      Generation: 0

[3] Meta Key: /Meta2/System/NodeLiveness
    Range r2:
      StartKey: /Meta2/""
      EndKey:   /System/NodeLiveness
      Replicas: (n1,s1):1
      Generation: 0

[4] Meta Key: /Meta2/System/NodeLivenessMax
    Range r3:
      StartKey: /System/NodeLiveness
      EndKey:   /System/NodeLivenessMax
      Replicas: (n1,s1):1
      Generation: 0
```

Fixes: #119421

Release note: None

Author: iskettaneh

pkg/ccl/logictestccl/testdata/logic_test/crdb_internal_tenant

@@ -251,12 +251,12 @@ txn_id  txn_fingerprint_id  query  implicit_txn  session_id  start_time  end_tim
 query ITTI
 SELECT range_id, start_pretty, end_pretty, lease_holder FROM crdb_internal.ranges
 ----
-79  /Tenant/10  /Tenant/11  1
+80  /Tenant/10  /Tenant/11  1
 
 query ITT
 SELECT range_id, start_pretty, end_pretty FROM crdb_internal.ranges_no_leases
 ----
-79  /Tenant/10  /Tenant/11
+80  /Tenant/10  /Tenant/11
 
 query IT
 SELECT zone_id, target FROM crdb_internal.zones ORDER BY 1

pkg/ccl/logictestccl/testdata/logic_test/multi_region_remote_access_error

@@ -471,7 +471,7 @@ skipif config multiregion-9node-3region-3azs-vec-off
 query I retry
 SELECT DISTINCT range_id FROM [SHOW RANGES FROM TABLE messages_rbr]
 ----
-84
+85
 
 # Update does not fail when accessing all rows in messages_rbr because lookup
 # join does not error out the lookup table in phase 1.

pkg/ccl/logictestccl/testdata/logic_test/regional_by_row_insert_fast_path

@@ -200,7 +200,7 @@ query T rowsort
 SELECT message FROM [SHOW KV TRACE FOR SESSION]
 WHERE message LIKE '%batch%' AND message LIKE '%Scan%'
 ----
-r78: sending batch 4 Scan to (n1,s1):1
+r79: sending batch 4 Scan to (n1,s1):1
 
 # Regression test for #115377.
 statement ok

pkg/cli/debug_recover_loss_of_quorum_test.go

@@ -153,7 +153,11 @@ func TestCollectInfoFromOnlineCluster(t *testing.T) {
 	require.Equal(t, 2, len(stores), "collected replicas from stores")
 	require.Equal(t, 2, len(replicas.LocalInfo), "collected info is not split by node")
 	require.Equal(t, totalRanges*2, totalReplicas, "number of collected replicas")
-	require.Equal(t, totalRanges, len(replicas.Descriptors),
+	// The number of range descriptors is counted by iterating over meta2 keys.
+	// Since meta1 and meta2 ranges are split, the number of range descriptors
+	// is going to be one less than the number of ranges as meta1 is a range but
+	// its descriptor isn't stored in meta2.
+	require.Equal(t, totalRanges-1, len(replicas.Descriptors),
 		"number of collected descriptors from metadata")
 	require.Equal(t, clusterversion.Latest.Version(), replicas.Version,
 		"collected version info from stores")

pkg/config/system.go

@@ -458,6 +458,7 @@ func (s *SystemConfig) getZoneEntry(codec keys.SQLCodec, id ObjectID) (zoneEntry
 }
 
 var staticSplits = []roachpb.RKey{
+	roachpb.RKey(keys.Meta2Prefix),                  // start of meta2 span
 	roachpb.RKey(keys.NodeLivenessPrefix),           // end of meta records / start of node liveness span
 	roachpb.RKey(keys.NodeLivenessKeyMax),           // end of node liveness span
 	roachpb.RKey(keys.TimeseriesPrefix),             // start of timeseries span

pkg/config/system_test.go

@@ -292,19 +292,19 @@ func TestComputeSplitKeySystemRanges(t *testing.T) {
 		start, end roachpb.RKey
 		split      roachpb.Key
 	}{
-		{roachpb.RKeyMin, roachpb.RKeyMax, keys.NodeLivenessPrefix},
-		{roachpb.RKeyMin, tkey(1), keys.NodeLivenessPrefix},
-		{roachpb.RKeyMin, roachpb.RKey(keys.TimeseriesPrefix), keys.NodeLivenessPrefix},
-		{roachpb.RKeyMin, roachpb.RKey(keys.SystemPrefix.Next()), nil},
-		{roachpb.RKeyMin, roachpb.RKey(keys.SystemPrefix), nil},
-		{roachpb.RKeyMin, roachpb.RKey(keys.MetaMax), nil},
-		{roachpb.RKeyMin, roachpb.RKey(keys.Meta2KeyMax), nil},
+		{roachpb.RKeyMin, roachpb.RKeyMax, keys.Meta2Prefix},
+		{roachpb.RKeyMin, tkey(1), keys.Meta2Prefix},
+		{roachpb.RKeyMin, roachpb.RKey(keys.TimeseriesPrefix), keys.Meta2Prefix},
+		{roachpb.RKeyMin, roachpb.RKey(keys.SystemPrefix.Next()), keys.Meta2Prefix},
+		{roachpb.RKeyMin, roachpb.RKey(keys.SystemPrefix), keys.Meta2Prefix},
+		{roachpb.RKeyMin, roachpb.RKey(keys.MetaMax), keys.Meta2Prefix},
+		{roachpb.RKeyMin, roachpb.RKey(keys.Meta2KeyMax), keys.Meta2Prefix},
 		{roachpb.RKeyMin, roachpb.RKey(keys.Meta1KeyMax), nil},
-		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKey(keys.SystemPrefix), nil},
-		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKey(keys.SystemPrefix.Next()), nil},
-		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKey(keys.NodeLivenessPrefix), nil},
-		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKey(keys.NodeLivenessPrefix.Next()), keys.NodeLivenessPrefix},
-		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKeyMax, keys.NodeLivenessPrefix},
+		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKey(keys.SystemPrefix), keys.Meta2Prefix},
+		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKey(keys.SystemPrefix.Next()), keys.Meta2Prefix},
+		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKey(keys.NodeLivenessPrefix), keys.Meta2Prefix},
+		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKey(keys.NodeLivenessPrefix.Next()), keys.Meta2Prefix},
+		{roachpb.RKey(keys.Meta1KeyMax), roachpb.RKeyMax, keys.Meta2Prefix},
 		{roachpb.RKey(keys.SystemPrefix), roachpb.RKey(keys.SystemPrefix), nil},
 		{roachpb.RKey(keys.SystemPrefix), roachpb.RKey(keys.SystemPrefix.Next()), nil},
 		{roachpb.RKey(keys.SystemPrefix), roachpb.RKeyMax, keys.NodeLivenessPrefix},

pkg/kv/kvprober/kvprober_integration_test.go

@@ -339,6 +339,12 @@ func TestPlannerMakesPlansCoveringAllRanges(t *testing.T) {
 			log.Dev.Infof(ctx, "current rangeID to times would be probed map: %v", rangeIDToTimesWouldBeProbed)
 
 			for i := int64(1); i <= numRanges; i++ {
+				// Range 1 (meta1 range) is not probed by the kvprober planner, as it
+				// only scans meta2 records, and range 1's descriptor isn't stored in
+				// meta2.
+				if i == 1 {
+					continue
+				}
 				// Expect all ranges to eventually be returned by Next n or n+1 times.
 				// Can't expect all ranges to be returned by Next exactly n times,
 				// as the order in which the lowest ordinal ranges are returned by

pkg/kv/kvserver/client_split_test.go

@@ -1945,15 +1945,19 @@ func runSetupSplitSnapshotRace(
 	// are up to date.
 	tc.WaitForValues(t, rightKey, []int64{0, 0, 0, 0 /* stopped */, 5, 5})
 
-	// Scan the meta ranges to resolve all intents
-	if _, pErr := kv.SendWrapped(context.Background(), tc.Servers[0].DistSenderI().(kv.Sender),
-		&kvpb.ScanRequest{
+	// Scan the meta ranges to resolve all intents.
+	if err := tc.Server(0).DB().Txn(context.Background(), func(ctx context.Context, txn *kv.Txn) error {
+		ba := &kvpb.BatchRequest{}
+		ba.Add(&kvpb.ScanRequest{
 			RequestHeader: kvpb.RequestHeader{
 				Key:    keys.MetaMin,
 				EndKey: keys.MetaMax,
 			},
-		}); pErr != nil {
-		t.Fatal(pErr)
+		})
+		_, pErr := txn.Send(ctx, ba)
+		return pErr.GoError()
+	}); err != nil {
+		t.Fatal(err)
 	}
 
 	// Stop the remaining data stores.

pkg/kv/kvserver/loqrecovery/server_integration_test.go

@@ -91,18 +91,23 @@ func TestReplicaCollection(t *testing.T) {
 
 		// Check counters on retrieved replica info.
 		cnt := getInfoCounters(replicas)
+
 		require.Equal(t, liveNodes, cnt.stores, "collected replicas from stores")
 		require.Equal(t, liveNodes, cnt.nodes, "collected replicas from nodes")
 		if expectRangeMeta {
-			require.Equal(t, totalRanges, cnt.descriptors,
+			// The number of range descriptors is counted by iterating over meta2
+			// keys. Since meta1 and meta2 ranges are split, the number of range
+			// descriptors is going to be one less than the number of ranges as meta1
+			// is a range but its descriptor isn't stored in meta2.
+			require.Equal(t, totalRanges, cnt.descriptors+1,
 				"number of collected descriptors from metadata")
 		}
 		require.Equal(t, totalRanges*liveNodes, cnt.replicas, "number of collected replicas")
 		// Check stats counters as well.
 		require.Equal(t, liveNodes, stats.Nodes, "node counter stats")
 		require.Equal(t, liveNodes, stats.Stores, "store counter stats")
 		if expectRangeMeta {
-			require.Equal(t, totalRanges, stats.Descriptors, "range descriptor counter stats")
+			require.Equal(t, totalRanges, stats.Descriptors+1, "range descriptor counter stats")
 		}
 		require.NotEqual(t, replicas.ClusterID, uuid.UUID{}.String(), "cluster UUID must not be empty")
 		require.Equal(t, replicas.Version,
@@ -166,14 +171,18 @@ func TestStreamRestart(t *testing.T) {
 		cnt := getInfoCounters(replicas)
 		require.Equal(t, liveNodes, cnt.stores, "collected replicas from stores")
 		require.Equal(t, liveNodes, cnt.nodes, "collected replicas from nodes")
-		require.Equal(t, totalRanges, cnt.descriptors,
+		// The number of range descriptors is counted by iterating over meta2
+		// keys. Since meta1 and meta2 ranges are split, the number of range
+		// descriptors is going to be one less than the number of ranges as meta1
+		// is a range but its descriptor isn't stored in meta2.
+		require.Equal(t, totalRanges, cnt.descriptors+1,
 			"number of collected descriptors from metadata")
 		require.Equal(t, totalRanges*liveNodes, cnt.replicas,
 			"number of collected replicas")
 		// Check stats counters as well.
 		require.Equal(t, liveNodes, stats.Nodes, "node counter stats")
 		require.Equal(t, liveNodes, stats.Stores, "store counter stats")
-		require.Equal(t, totalRanges, stats.Descriptors, "range descriptor counter stats")
+		require.Equal(t, totalRanges, stats.Descriptors+1, "range descriptor counter stats")
 	}
 
 	assertReplicas(3)

pkg/kv/kvserver/queue_test.go

@@ -721,7 +721,7 @@ func TestAcceptsUnsplitRanges(t *testing.T) {
 		},
 		stopper)
 
-	maxWontSplitAddr, err := keys.Addr(keys.SystemPrefix)
+	maxWontSplitAddr, err := keys.Addr(keys.Meta1KeyMax)
 	require.NoError(t, err)
 
 	minWillSplitAddr, err := keys.Addr(keys.TableDataMin)