asim: use cluster setting LBRebalancingMode directly

Previously, LBRebalancingMode existed both as a simulation setting and a cluster
setting, which was confusing especially now that we're plumbing the cluster
setting through. This commit removes the simulation specific setting and makes
the simulator use the cluster setting directly instead. Note that default
behavior is leases mode and replicas=2 remains unchanged for both.

Epic: none
Release note: none

Author: wenyihu6

pkg/kv/kvserver/asim/config/settings.go

@@ -27,7 +27,6 @@ const (
 	defaultSplitQPSThreshold       = 2500
 	defaultSplitStatRetention      = 10 * time.Minute
 	defaultSeed                    = 42
-	defaultLBRebalancingMode       = 2 // Leases and replicas.
 	defaultLBRebalancingInterval   = time.Minute
 	defaultLBRebalanceQPSThreshold = 0.1
 	defaultLBRebalancingObjective  = 0 // QPS
@@ -96,9 +95,6 @@ type SimulationSettings struct {
 	// SplitStatRetention is the duration which recorded load will be retained
 	// and factored into load based splitting decisions.
 	SplitStatRetention time.Duration
-	// LBRebalancingMode controls if and when we do store-level rebalancing
-	// based on load. It maps to kvserver.LBRebalancingMode.
-	LBRebalancingMode int64
 	// LBRebalancingObjective is the load objective to balance.
 	LBRebalancingObjective int64
 	// LBRebalancingInterval controls how often the store rebalancer will
@@ -136,7 +132,6 @@ func DefaultSimulationSettings() *SimulationSettings {
 		StateExchangeDelay:      defaultStateExchangeDelay,
 		SplitQPSThreshold:       defaultSplitQPSThreshold,
 		SplitStatRetention:      defaultSplitStatRetention,
-		LBRebalancingMode:       defaultLBRebalancingMode,
 		LBRebalancingObjective:  defaultLBRebalancingObjective,
 		LBRebalancingInterval:   defaultLBRebalancingInterval,
 		ReplicateQueueEnabled:   true,

pkg/kv/kvserver/asim/event/mutation_event.go

@@ -54,8 +54,9 @@ type SetNodeLocalityEvent struct {
 // SetSimulationSettingsEvent represents a mutation event responsible for
 // changing a simulation setting during the simulation.
 type SetSimulationSettingsEvent struct {
-	Key   string
-	Value interface{}
+	IsClusterSetting bool
+	Key              string
+	Value            interface{}
 }
 
 var _ Event = &SetSpanConfigEvent{}
@@ -140,7 +141,11 @@ func (sne SetNodeLocalityEvent) String() string {
 
 func (se SetSimulationSettingsEvent) Func() EventFunc {
 	return MutationFunc(func(ctx context.Context, s state.State) {
-		s.SetSimulationSettings(se.Key, se.Value)
+		if se.IsClusterSetting {
+			s.SetClusterSetting(se.Key, se.Value)
+		} else {
+			s.SetSimulationSettings(se.Key, se.Value)
+		}
 	})
 }
 

pkg/kv/kvserver/asim/mmaintegration/mma_store_rebalancer.go

@@ -97,7 +97,7 @@ func (msr *MMAStoreRebalancer) Tick(ctx context.Context, tick time.Time, s state
 		return
 	}
 
-	if msr.settings.LBRebalancingMode != int64(kvserver.LBRebalancingMultiMetric) {
+	if kvserver.LoadBasedRebalancingMode.Get(&msr.settings.ST.SV) != kvserver.LBRebalancingMultiMetric {
 		// When the store rebalancer isn't set to use the multi-metric mode, the
 		// legacy store rebalancer is used.
 		return

pkg/kv/kvserver/asim/state/impl.go

@@ -1393,6 +1393,15 @@ func (s *state) RegisterConfigChangeListener(listener ConfigChangeListener) {
 	s.configChangeListeners = append(s.configChangeListeners, listener)
 }
 
+func (s *state) SetClusterSetting(Key string, Value interface{}) {
+	switch Key {
+	case "LBRebalancingMode":
+		kvserver.LoadBasedRebalancingMode.Override(context.Background(), &s.settings.ST.SV, kvserver.LBRebalancingMode(Value.(int64)))
+	default:
+		panic("other cluster settings not supported")
+	}
+}
+
 // SetSimulationSettings sets the simulation setting for the given key to the
 // given value.
 func (s *state) SetSimulationSettings(Key string, Value interface{}) {

pkg/kv/kvserver/asim/state/state.go

@@ -202,6 +202,8 @@ type State interface {
 	RegisterConfigChangeListener(ConfigChangeListener)
 	// SetSimulationSettings sets the simulation settings for the state.
 	SetSimulationSettings(Key string, Value interface{})
+	// SetClusterSetting sets the cluster setting for the state.
+	SetClusterSetting(Key string, Value interface{})
 	// NodeCapacity returns the capacity of the node with ID NodeID.
 	NodeCapacity(NodeID) roachpb.NodeCapacity
 	// SetNodeCPURateCapacity sets the CPU rate capacity for the node with ID

pkg/kv/kvserver/asim/storerebalancer/store_rebalancer.go

@@ -190,7 +190,7 @@ func (src *storeRebalancerControl) phasePrologue(
 			s, src.storeID,
 			kvserver.LBRebalancingObjective(src.settings.LBRebalancingObjective).ToDimension(),
 		),
-		kvserver.LBRebalancingMode(src.settings.LBRebalancingMode),
+		kvserver.LoadBasedRebalancingMode.Get(&src.settings.ST.SV),
 	)
 
 	if !src.sr.ShouldRebalanceStore(ctx, rctx) {

pkg/kv/kvserver/asim/tests/BUILD.bazel

@@ -38,6 +38,7 @@ go_test(
     data = glob(["testdata/**"]),
     embed = [":tests"],
     deps = [
+        "//pkg/kv/kvserver",
         "//pkg/kv/kvserver/allocator/allocatorimpl",
         "//pkg/kv/kvserver/asim/assertion",
         "//pkg/kv/kvserver/asim/config",

pkg/kv/kvserver/asim/tests/datadriven_simulation_test.go

@@ -13,6 +13,7 @@ import (
 	"testing"
 	"time"
 
+	"github.com/cockroachdb/cockroach/pkg/kv/kvserver"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/allocator/allocatorimpl"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/assertion"
 	"github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/config"
@@ -509,25 +510,29 @@ func TestDataDriven(t *testing.T) {
 				}
 				return ""
 			case "setting":
+				scanIfExists(t, d, "replicate_queue_enabled", &settingsGen.Settings.ReplicateQueueEnabled)
+				scanIfExists(t, d, "lease_queue_enabled", &settingsGen.Settings.LeaseQueueEnabled)
+				scanIfExists(t, d, "split_queue_enabled", &settingsGen.Settings.SplitQueueEnabled)
+				scanIfExists(t, d, "rebalance_interval", &settingsGen.Settings.LBRebalancingInterval)
+				scanIfExists(t, d, "split_qps_threshold", &settingsGen.Settings.SplitQPSThreshold)
+				scanIfExists(t, d, "rebalance_range_threshold", &settingsGen.Settings.RangeRebalanceThreshold)
+				scanIfExists(t, d, "gossip_delay", &settingsGen.Settings.StateExchangeDelay)
+				scanIfExists(t, d, "range_size_split_threshold", &settingsGen.Settings.RangeSizeSplitThreshold)
+				scanIfExists(t, d, "rebalance_objective", &settingsGen.Settings.LBRebalancingObjective)
 				var delay time.Duration
 				if isDelayed := scanIfExists(t, d, "delay", &delay); isDelayed {
 					var rebalanceMode int64
 					scanIfExists(t, d, "rebalance_mode", &rebalanceMode)
 					eventGen.ScheduleEvent(settingsGen.Settings.StartTime, delay, event.SetSimulationSettingsEvent{
-						Key:   "LBRebalancingMode",
-						Value: rebalanceMode,
+						IsClusterSetting: true,
+						Key:              "LBRebalancingMode",
+						Value:            rebalanceMode,
 					})
 				} else {
-					scanIfExists(t, d, "replicate_queue_enabled", &settingsGen.Settings.ReplicateQueueEnabled)
-					scanIfExists(t, d, "lease_queue_enabled", &settingsGen.Settings.LeaseQueueEnabled)
-					scanIfExists(t, d, "split_queue_enabled", &settingsGen.Settings.SplitQueueEnabled)
-					scanIfExists(t, d, "rebalance_mode", &settingsGen.Settings.LBRebalancingMode)
-					scanIfExists(t, d, "rebalance_interval", &settingsGen.Settings.LBRebalancingInterval)
-					scanIfExists(t, d, "split_qps_threshold", &settingsGen.Settings.SplitQPSThreshold)
-					scanIfExists(t, d, "rebalance_range_threshold", &settingsGen.Settings.RangeRebalanceThreshold)
-					scanIfExists(t, d, "gossip_delay", &settingsGen.Settings.StateExchangeDelay)
-					scanIfExists(t, d, "range_size_split_threshold", &settingsGen.Settings.RangeSizeSplitThreshold)
-					scanIfExists(t, d, "rebalance_objective", &settingsGen.Settings.LBRebalancingObjective)
+					var rebalanceMode int64
+					if exists := scanIfExists(t, d, "rebalance_mode", &rebalanceMode); exists {
+						kvserver.LoadBasedRebalancingMode.Override(ctx, &settingsGen.Settings.ST.SV, kvserver.LBRebalancingMode(rebalanceMode))
+					}
 				}
 				return ""
 			case "print":