Merge #154513

154513: asim: print generated load, trace StoreRebalancer r=wenyihu6 a=tbg

See individual commits.

Epic: CRDB-49117

Co-authored-by: Tobias Grieger <[email protected]>
Co-authored-by: wenyihu6 <[email protected]>

Author: 3 people

pkg/kv/kvserver/asim/asim.go

@@ -98,13 +98,18 @@ func NewSimulator(
 	changer := state.NewReplicaChanger()
 	controllers := make(map[state.StoreID]op.Controller)
 
+	var onRecording func(storeID state.StoreID, atDuration time.Duration, rec tracingpb.Recording)
+	if fn := settings.OnRecording; fn != nil {
+		onRecording = func(storeID state.StoreID, atDuration time.Duration, rec tracingpb.Recording) {
+			fn(int64(storeID), atDuration, rec)
+		}
+	}
+
 	s := &Simulator{
 		AmbientContext: log.MakeTestingAmbientCtxWithNewTracer(),
-		onRecording: func(storeID state.StoreID, atDuration time.Duration, rec tracingpb.Recording) {
-			if fn := settings.OnRecording; fn != nil {
-				fn(int64(storeID), atDuration, rec)
-			}
-		},
+		// onRecording is intentionally nil if settings.OnRecording is nil, to
+		// short-circuit trace creation overhead in that case.
+		onRecording: onRecording,
 		curr:        settings.StartTime,
 		end:         settings.StartTime.Add(duration),
 		interval:    settings.TickInterval,
@@ -404,7 +409,9 @@ func (s *Simulator) tickStoreRebalancers(ctx context.Context, tick time.Time, st
 	stores := s.state.Stores()
 	s.shuffler(len(stores), func(i, j int) { stores[i], stores[j] = stores[j], stores[i] })
 	for _, store := range stores {
-		s.srs[store.StoreID()].Tick(ctx, tick, state)
+		s.doAndMaybeTrace(ctx, store.StoreID(), tick, "StoreRebalancer", func(ctx context.Context) {
+			s.srs[store.StoreID()].Tick(ctx, tick, state)
+		})
 	}
 }
 

pkg/kv/kvserver/asim/tests/datadriven_simulation_test.go

@@ -244,15 +244,17 @@ func TestDataDriven(t *testing.T) {
 					// CPU consumptions. This isn't exact because it doesn't account for
 					// replication, but it's close enough.
 					// NB: writes also consume requestCPUPerAccess.
-					approxVCPUs := (rate * (float64(requestCPUPerAccess) + float64(raftCPUPerAccess)*(1.0-rwRatio))) / 1e9
+					accessVCPUs := rate * float64(requestCPUPerAccess) / 1e9
+					beforeReplicationRaftVCPUs := rate * (1 - rwRatio) * float64(raftCPUPerAccess) / 1e9
+					approxVCPUs := accessVCPUs + beforeReplicationRaftVCPUs
 					// Ditto for writes. Here too we don't account for replication. Note
 					// that at least under uniform writes, real clusters can have a write
 					// amp that easily surpasses 20, so writing at 40mb/s to a small set
 					// of stores would often constitute an issue in production.
-					approxWriteBytes := float64(maxBlock+minBlock) * rate * (1.0 - rwRatio) / 2
+					beforeReplicationWriteBytes := float64(maxBlock+minBlock) * rate * (1.0 - rwRatio) / 2
 
 					const tenkb = 10 * 1024
-					neitherWriteNorCPUHeavy := approxWriteBytes < tenkb && approxVCPUs < .5
+					neitherWriteNorCPUHeavy := beforeReplicationWriteBytes < tenkb && approxVCPUs < .5
 
 					// We tolerate abnormally low CPU if there's a sensible amount of
 					// write load. Otherwise, it's likely a mistake.
@@ -261,9 +263,24 @@ func TestDataDriven(t *testing.T) {
 					}
 					// Similarly, tolerate abnormally low write load when there's
 					// significant CPU. Independently, call out high write load.
-					if (neitherWriteNorCPUHeavy && approxWriteBytes > 0) || approxWriteBytes > 40*(1<<20) {
+					if (neitherWriteNorCPUHeavy && beforeReplicationWriteBytes > 0) || beforeReplicationWriteBytes > 40*(1<<20) {
 						_, _ = fmt.Fprintf(&buf, "WARNING: write load of %s is likely accidental\n",
-							humanizeutil.IBytes(int64(approxWriteBytes)))
+							humanizeutil.IBytes(int64(beforeReplicationWriteBytes)))
+					}
+					{
+						var parts []string
+						if accessVCPUs > 0 {
+							parts = append(parts, fmt.Sprintf("%.2f access-vcpus", accessVCPUs))
+						}
+						if beforeReplicationRaftVCPUs > 0 {
+							parts = append(parts, fmt.Sprintf("%.2f raft-vcpus", beforeReplicationRaftVCPUs))
+						}
+						if beforeReplicationWriteBytes > 0 {
+							parts = append(parts, fmt.Sprintf("%s/s goodput", humanizeutil.IBytes(int64(beforeReplicationWriteBytes))))
+						}
+						if len(parts) > 0 {
+							_, _ = fmt.Fprintln(&buf, strings.Join(parts, ", "))
+						}
 					}
 
 					var nextLoadGen gen.BasicLoad
@@ -560,8 +577,12 @@ func TestDataDriven(t *testing.T) {
 							seedGen := rand.New(rand.NewSource(seed))
 							for sample := 0; sample < samples; sample++ {
 								tr := makeTraceHelper(rewrite, plotDir, testName, sample+1, duration)
-								settingsGen.Settings.OnRecording = func(storeID int64, atDuration time.Duration, rec tracingpb.Recording) {
-									tr.OnRecording(t, storeID, atDuration, rec)
+								if tr.enabled {
+									// Only populate OnRecording if we're going to save the results.
+									// That way, we avoid creating trace spans during normal test runs.
+									settingsGen.Settings.OnRecording = func(storeID int64, atDuration time.Duration, rec tracingpb.Recording) {
+										tr.OnRecording(t, storeID, atDuration, rec)
+									}
 								}
 
 								assertionFailures := []string{}

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/heterogeneous_cpu.txt

@@ -21,6 +21,7 @@ gen_ranges ranges=200 min_key=1 max_key=10000 placement_type=even
 # cpu nanos per second.
 gen_load rate=40000 rw_ratio=1 request_cpu_per_access=500000 min_key=1 max_key=10000
 ----
+20.00 access-vcpus
 
 # We want the CPU load to balance based on %cpu. But both the MMA and SMA balance
 # on absolute cpu-nanos, i.e. not taking into account that n3 has double the capacity.

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu.txt

@@ -22,16 +22,20 @@ setting split_queue_enabled=false
 gen_ranges ranges=100 min_key=0 max_key=10000
 ----
 
+# TODO(tbg): likely accidentally too low.
 gen_load rate=5000 rw_ratio=0.95 min_block=100 max_block=100 request_cpu_per_access=100 raft_cpu_per_write=20 min_key=0 max_key=10000
 ----
+0.00 access-vcpus, 0.00 raft-vcpus, 24 KiB/s goodput
 
 # Another workload is added over the second half of the keyspace, which is initially
 # mostly on s1-s3.
 gen_ranges ranges=50 min_key=10001 max_key=20000 placement_type=skewed
 ----
 
+# TODO(tbg): likely accidentally too low.
 gen_load rate=5000 rw_ratio=0.95 min_block=128 max_block=128 request_cpu_per_access=100000 raft_cpu_per_write=20000 min_key=10001 max_key=20000
 ----
+0.50 access-vcpus, 0.01 raft-vcpus, 31 KiB/s goodput
 
 eval duration=2m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,replicas,leases)
 ----

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_25nodes.txt

@@ -12,8 +12,10 @@ setting split_queue_enabled=false
 gen_ranges ranges=50 min_key=0 max_key=10000
 ----
 
+# TODO(tbg): likely accidentally too low.
 gen_load rate=15000 rw_ratio=0.95 min_block=100 max_block=100 request_cpu_per_access=100 raft_cpu_per_write=20 min_key=0 max_key=10000
 ----
+0.00 access-vcpus, 0.00 raft-vcpus, 73 KiB/s goodput
 
 # Another workload is added over the second half of the keyspace, which is initially
 # only mainly on s1-s3 due to the skewed distribution.
@@ -22,6 +24,7 @@ gen_ranges ranges=50 min_key=10001 max_key=20000 placement_type=skewed
 
 gen_load rate=15000 rw_ratio=0.95 min_block=1 max_block=1 request_cpu_per_access=7000000 raft_cpu_per_write=20000 min_key=10001 max_key=20000
 ----
+105.00 access-vcpus, 0.02 raft-vcpus, 750 B/s goodput
 
 eval duration=20m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_able_to_shed_leases.txt

@@ -23,6 +23,7 @@ gen_ranges ranges=25 min_key=0 max_key=10000 placement_type=replica_placement
 
 gen_load rate=50000 rw_ratio=0 min_key=0 max_key=10000 raft_cpu_per_write=100000
 ----
+5.00 raft-vcpus, 49 KiB/s goodput
 
 eval duration=5m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_unable_to_shed_leases.txt

@@ -35,6 +35,7 @@ gen_ranges ranges=25 min_key=0 max_key=10000 placement_type=replica_placement
 
 gen_load rate=5000 rw_ratio=0 min_key=0 max_key=10000 raft_cpu_per_write=1000000
 ----
+5.00 raft-vcpus, 4.9 KiB/s goodput
 
 eval duration=5m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_write_uniform_cpu.txt

@@ -13,6 +13,7 @@ gen_ranges ranges=30 min_key=1 max_key=10000 placement_type=even
 
 gen_load rate=1000 rw_ratio=1.0 request_cpu_per_access=5000000 min_key=1 max_key=10000
 ----
+5.00 access-vcpus
 
 # Write only workload, which generates no CPU and 20000op/s*1000B/op =
 # 20000000B/s (x 3 replication factor) write bytes per second over the second half
@@ -22,6 +23,7 @@ gen_ranges ranges=30 min_key=10001 max_key=20000 placement_type=skewed
 
 gen_load rate=20000 rw_ratio=0 min_block=1000 max_block=1000 min_key=10001 max_key=20000
 ----
+19 MiB/s goodput
 
 setting split_queue_enabled=false
 ----

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/one_voter_skewed_cpu_skewed_write.txt

@@ -19,13 +19,17 @@ gen_ranges ranges=100 repl_factor=1 min_key=10001 max_key=20000 placement_type=r
 {s2:*}:1
 
 # read cpu load of 1000x100=10k, all hitting s1, which is then at 100% cpu.
+# TODO(tbg): the CPU count is accidentally too low.
 gen_load rate=1000 rw_ratio=1.0 request_cpu_per_access=500000 min_key=1 max_key=10000
 ----
+0.50 access-vcpus
 
 # Write only workload, which generates 20% cpu and 5mb of writes per second.
 # over the second half of the keyspace.
+# TODO(tbg): the CPU load is too low.
 gen_load rate=5000 rw_ratio=0 min_block=1000 max_block=1000 raft_cpu_per_write=1 min_key=10001 max_key=20000
 ----
+0.00 raft-vcpus, 4.8 MiB/s goodput
 
 setting split_queue_enabled=false
 ----

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/skewed_cpu_even_ranges_mma.txt

@@ -20,6 +20,7 @@ gen_ranges ranges=36 min_key=1 max_key=10000 placement_type=replica_placement by
 # 5ms of request CPU per access and 500µs of raft CPU per write @ 1000/s.
 gen_load rate=1000 rw_ratio=0.95 min_block=100 max_block=100 request_cpu_per_access=5000000 raft_cpu_per_write=500000 min_key=1 max_key=10000
 ----
+5.00 access-vcpus, 0.03 raft-vcpus, 4.9 KiB/s goodput
 
 # Almost empty workload, which generates no CPU and small amount of writes
 # over the second half of the keyspace, scattered over s4-s9.
@@ -30,8 +31,10 @@ gen_ranges ranges=72 min_key=10001 max_key=20000 placement_type=replica_placemen
 {s4:*,s5,s6}:1
 {s7:*,s8,s9}:1
 
+# TODO(tbg): this is barely anything, is this intentional?
 gen_load rate=100 rw_ratio=0 min_block=128 max_block=128 min_key=10001 max_key=20000
 ----
+12 KiB/s goodput
 
 setting split_queue_enabled=false
 ----