asim: scale up cpu load in gen_load

This commit adjusts some cpu load config for gen_load (e.g.,
request_cpu_per_access) to maintain a reasonable cpu utilization in some of the
asim dd setup. Some values were too low since the beginning. But recent changes
to scale up node_cpu_rate_capacity made things worse. This is important because
mma only triggers cpu rebalancing when cpu util exceeds a certain threshold.
https://github.com/cockroachdb/cockroach/blob/93f41d0e0dabdf54c7275581e0767452d6df5390/pkg/kv/kvserver/allocator/mmaprototype/load.go#L531

Author: wenyihu6

pkg/kv/kvserver/asim/tests/testdata/non_rand/load_distribution_movement_disabled_enable_later.txt

@@ -34,7 +34,7 @@ gen_ranges ranges=30 min_key=10001 max_key=20000 placement_type=replica_placemen
 {s10:*,s6,s7}:1
 
 # Read-only workload on first 30 ranges. 5 cores.
-gen_load rate=1000000 rw_ratio=1 request_cpu_per_access=5000 min_key=1 max_key=10000
+gen_load rate=1000000 rw_ratio=1 request_cpu_per_access=50000 min_key=1 max_key=10000
 ----
 
 # Write only workload on second 30 ranges. 10mb/s before 3x replication.
@@ -47,17 +47,17 @@ setting rebalance_mode=4 delay=2m
 
 eval duration=7m samples=1 seed=42 cfgs=(sma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
-cpu#1: last:  [s1=451007155, s2=424066775, s3=481923720, s4=487552810, s5=432089126, s6=536127090, s7=582915883, s8=497226435, s9=545078443, s10=559276708] (stddev=51954473.68, mean=499726414.50, sum=4997264145)
-cpu#1: thrash_pct: [s1=19%, s2=33%, s3=40%, s4=24%, s5=15%, s6=16%, s7=14%, s8=7%, s9=12%, s10=14%]  (sum=193%)
-cpu_util#1: last:  [s1=0.03, s2=0.03, s3=0.03, s4=0.03, s5=0.03, s6=0.03, s7=0.04, s8=0.03, s9=0.03, s10=0.03] (stddev=0.00, mean=0.03, sum=0)
-cpu_util#1: thrash_pct: [s1=19%, s2=33%, s3=40%, s4=24%, s5=15%, s6=16%, s7=14%, s8=7%, s9=12%, s10=14%]  (sum=193%)
+cpu#1: last:  [s1=5081867763, s2=4625570068, s3=4683450985, s4=5175660716, s5=4480723281, s6=5400432969, s7=5232941604, s8=5060159898, s9=5285669166, s10=4934994414] (stddev=291808172.88, mean=4996147086.40, sum=49961470864)
+cpu#1: thrash_pct: [s1=29%, s2=33%, s3=54%, s4=33%, s5=22%, s6=6%, s7=35%, s8=14%, s9=18%, s10=32%]  (sum=275%)
+cpu_util#1: last:  [s1=0.32, s2=0.29, s3=0.29, s4=0.32, s5=0.28, s6=0.34, s7=0.33, s8=0.32, s9=0.33, s10=0.31] (stddev=0.02, mean=0.31, sum=3)
+cpu_util#1: thrash_pct: [s1=29%, s2=33%, s3=54%, s4=33%, s5=22%, s6=6%, s7=35%, s8=14%, s9=18%, s10=32%]  (sum=275%)
 leases#1: first: [s1=6, s2=6, s3=6, s4=6, s5=6, s6=6, s7=6, s8=6, s9=6, s10=6] (stddev=0.00, mean=6.00, sum=60)
-leases#1: last:  [s1=54, s2=56, s3=55, s4=53, s5=57, s6=58, s7=60, s8=57, s9=60, s10=61] (stddev=2.55, mean=57.10, sum=571)
-leases#1: thrash_pct: [s1=106%, s2=118%, s3=95%, s4=95%, s5=123%, s6=33%, s7=33%, s8=27%, s9=27%, s10=30%]  (sum=688%)
+leases#1: last:  [s1=56, s2=56, s3=56, s4=56, s5=55, s6=59, s7=58, s8=62, s9=58, s10=55] (stddev=2.07, mean=57.10, sum=571)
+leases#1: thrash_pct: [s1=115%, s2=115%, s3=120%, s4=101%, s5=140%, s6=24%, s7=53%, s8=30%, s9=33%, s10=56%]  (sum=787%)
 replicas#1: first: [s1=18, s2=18, s3=18, s4=18, s5=18, s6=18, s7=18, s8=18, s9=18, s10=18] (stddev=0.00, mean=18.00, sum=180)
-replicas#1: last:  [s1=175, s2=172, s3=179, s4=173, s5=173, s6=174, s7=167, s8=169, s9=169, s10=162] (stddev=4.50, mean=171.30, sum=1713)
-replicas#1: thrash_pct: [s1=87%, s2=91%, s3=92%, s4=93%, s5=98%, s6=16%, s7=15%, s8=14%, s9=15%, s10=10%]  (sum=531%)
-write_bytes_per_second#1: last:  [s1=2994621, s2=2991747, s3=3008247, s4=2988902, s5=3006699, s6=2993336, s7=2993993, s8=3002728, s9=2996629, s10=2997216] (stddev=6108.70, mean=2997411.80, sum=29974118)
-write_bytes_per_second#1: thrash_pct: [s1=73%, s2=40%, s3=85%, s4=74%, s5=5%, s6=160%, s7=207%, s8=103%, s9=114%, s10=55%]  (sum=916%)
-artifacts[sma-count]: ae2919ade8043626
+replicas#1: last:  [s1=170, s2=176, s3=174, s4=175, s5=172, s6=166, s7=172, s8=170, s9=170, s10=168] (stddev=2.97, mean=171.30, sum=1713)
+replicas#1: thrash_pct: [s1=91%, s2=91%, s3=105%, s4=93%, s5=102%, s6=16%, s7=17%, s8=18%, s9=15%, s10=16%]  (sum=564%)
+write_bytes_per_second#1: last:  [s1=2661323, s2=2656875, s3=4334085, s4=3343904, s5=2658933, s6=2329852, s7=2329103, s8=3684224, s9=3345882, s10=2662193] (stddev=620316.35, mean=3000637.40, sum=30006374)
+write_bytes_per_second#1: thrash_pct: [s1=128%, s2=93%, s3=155%, s4=167%, s5=79%, s6=173%, s7=220%, s8=204%, s9=169%, s10=169%]  (sum=1557%)
+artifacts[sma-count]: b796f1485279b21d
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma_constraint_satisfaction1.txt

@@ -18,7 +18,7 @@ gen_ranges ranges=100 repl_factor=5 placement_type=replica_placement
 ----
 {s1:*,s2,s4,s5,s6}:1
 
-gen_load rate=1000 rw_ratio=0.95 min_block=1000 max_block=1000 request_cpu_per_access=100000 raft_cpu_per_write=10000
+gen_load rate=1000 rw_ratio=0.95 min_block=1000 max_block=1000 request_cpu_per_access=5000000 raft_cpu_per_write=10000
 ----
 
 set_span_config
@@ -30,15 +30,15 @@ setting split_queue_enabled=false
 
 eval duration=40m samples=1 seed=42 cfgs=(mma-only) metrics=(cpu,cpu_util,leases,replicas)
 ----
-cpu#1: last:  [s1=100533499, s2=500166, s3=0, s4=500166, s5=500166, s6=500166, s7=0, s8=0, s9=0] (stddev=31516919.07, mean=11392684.78, sum=102534163)
-cpu#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)
-cpu_util#1: last:  [s1=0.02, s2=0.00, s3=0.00, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00] (stddev=0.01, mean=0.00, sum=0)
-cpu_util#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)
+cpu#1: last:  [s1=1796346149, s2=1601160605, s3=1602033353, s4=499258, s5=499258, s6=499258, s7=0, s8=0, s9=0] (stddev=787270621.50, mean=555670875.67, sum=5001037881)
+cpu#1: thrash_pct: [s1=8%, s2=31%, s3=31%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=70%)
+cpu_util#1: last:  [s1=0.36, s2=0.32, s3=0.32, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00] (stddev=0.16, mean=0.11, sum=1)
+cpu_util#1: thrash_pct: [s1=8%, s2=31%, s3=31%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=70%)
 leases#1: first: [s1=100, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=31.43, mean=11.11, sum=100)
-leases#1: last:  [s1=100, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=31.43, mean=11.11, sum=100)
+leases#1: last:  [s1=36, s2=32, s3=32, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=15.75, mean=11.11, sum=100)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)
 replicas#1: first: [s1=100, s2=100, s3=0, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=49.69, mean=55.56, sum=500)
-replicas#1: last:  [s1=100, s2=100, s3=0, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=49.69, mean=55.56, sum=500)
+replicas#1: last:  [s1=68, s2=100, s3=32, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=44.56, mean=55.56, sum=500)
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)
-artifacts[mma-only]: 621f3d4f69649ede
+artifacts[mma-only]: 29c929f977fd070d
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma_constraint_satisfaction_old_alloc.txt

@@ -18,7 +18,7 @@ gen_ranges ranges=100 repl_factor=5 placement_type=replica_placement
 ----
 {s1:*,s2,s4,s5,s6}:1
 
-gen_load rate=1000 rw_ratio=0.95 min_block=1000 max_block=1000 request_cpu_per_access=100000 raft_cpu_per_write=10000
+gen_load rate=2000 rw_ratio=0.95 min_block=1000 max_block=1000 request_cpu_per_access=1000000 raft_cpu_per_write=10000
 ----
 
 set_span_config
@@ -27,27 +27,27 @@ set_span_config
 
 eval duration=60m samples=1 seed=42 metrics=(cpu,cpu_util,leases,replicas) cfgs=(sma-count,mma-only)
 ----
-cpu#1: last:  [s1=41374276, s2=27297814, s3=32347286, s4=499562, s5=499562, s6=499562, s7=0, s8=0, s9=0] (stddev=16111846.58, mean=11390895.78, sum=102518062)
-cpu#1: thrash_pct: [s1=14%, s2=14%, s3=29%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=59%)
-cpu_util#1: last:  [s1=0.01, s2=0.01, s3=0.01, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00] (stddev=0.00, mean=0.00, sum=0)
-cpu_util#1: thrash_pct: [s1=14%, s2=14%, s3=29%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=59%)
+cpu#1: last:  [s1=720317962, s2=660763359, s3=621361879, s4=999944, s5=999944, s6=999944, s7=0, s8=0, s9=0] (stddev=315294026.04, mean=222827003.56, sum=2005443032)
+cpu#1: thrash_pct: [s1=10%, s2=11%, s3=33%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=55%)
+cpu_util#1: last:  [s1=0.14, s2=0.13, s3=0.12, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00] (stddev=0.06, mean=0.04, sum=0)
+cpu_util#1: thrash_pct: [s1=10%, s2=11%, s3=33%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=55%)
 leases#1: first: [s1=100, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=31.43, mean=11.11, sum=100)
-leases#1: last:  [s1=41, s2=27, s3=32, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=16.07, mean=11.11, sum=100)
-leases#1: thrash_pct: [s1=0%, s2=0%, s3=8%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=8%)
+leases#1: last:  [s1=36, s2=33, s3=31, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=15.76, mean=11.11, sum=100)
+leases#1: thrash_pct: [s1=0%, s2=3%, s3=19%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=22%)
 replicas#1: first: [s1=100, s2=100, s3=0, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=49.69, mean=55.56, sum=500)
-replicas#1: last:  [s1=69, s2=67, s3=64, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=41.59, mean=55.56, sum=500)
-replicas#1: thrash_pct: [s1=8%, s2=6%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=14%)
-artifacts[sma-count]: cb26954cf324c27d
+replicas#1: last:  [s1=69, s2=66, s3=65, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=41.59, mean=55.56, sum=500)
+replicas#1: thrash_pct: [s1=10%, s2=8%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=18%)
+artifacts[sma-count]: 5a2ae3dcaecef53f
 ==========================
-cpu#1: last:  [s1=100533500, s2=500166, s3=0, s4=500166, s5=500166, s6=500166, s7=0, s8=0, s9=0] (stddev=31516919.38, mean=11392684.89, sum=102534164)
-cpu#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)
-cpu_util#1: last:  [s1=0.02, s2=0.00, s3=0.00, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00] (stddev=0.01, mean=0.00, sum=0)
-cpu_util#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)
+cpu#1: last:  [s1=719651325, s2=641735273, s3=641140178, s4=1000177, s5=1000177, s6=1000177, s7=0, s8=0, s9=0] (stddev=315151021.86, mean=222836367.44, sum=2005527307)
+cpu#1: thrash_pct: [s1=7%, s2=23%, s3=22%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=53%)
+cpu_util#1: last:  [s1=0.14, s2=0.13, s3=0.13, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00] (stddev=0.06, mean=0.04, sum=0)
+cpu_util#1: thrash_pct: [s1=7%, s2=23%, s3=22%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=53%)
 leases#1: first: [s1=100, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=31.43, mean=11.11, sum=100)
-leases#1: last:  [s1=100, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=31.43, mean=11.11, sum=100)
+leases#1: last:  [s1=36, s2=32, s3=32, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=15.75, mean=11.11, sum=100)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)
 replicas#1: first: [s1=100, s2=100, s3=0, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=49.69, mean=55.56, sum=500)
-replicas#1: last:  [s1=100, s2=100, s3=0, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=49.69, mean=55.56, sum=500)
+replicas#1: last:  [s1=68, s2=100, s3=32, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=44.56, mean=55.56, sum=500)
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)
-artifacts[mma-only]: 64926c4817b68708
+artifacts[mma-only]: a6c9461b2f3a0720
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma_high_cpu_25nodes.txt

@@ -19,22 +19,22 @@ gen_load rate=15000 rw_ratio=0.95 min_block=100 max_block=100 request_cpu_per_ac
 gen_ranges ranges=50 min_key=10001 max_key=20000 placement_type=skewed
 ----
 
-gen_load rate=15000 rw_ratio=0.95 min_block=1 max_block=1 request_cpu_per_access=100000 raft_cpu_per_write=20000 min_key=10001 max_key=20000
+gen_load rate=15000 rw_ratio=0.95 min_block=1 max_block=1 request_cpu_per_access=7000000 raft_cpu_per_write=20000 min_key=10001 max_key=20000
 ----
 
 eval duration=25m samples=1 seed=42 cfgs=(mma-only) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
-cpu#1: last:  [s1=374886931, s2=371179870, s3=245540920, s4=93235414, s5=91824370, s6=31024426, s7=656167, s8=30339076, s9=359653, s10=30460955, s11=388064, s12=30340742, s13=394468, s14=30693430, s15=359565, s16=30532188, s17=30405056, s18=60714988, s19=360175, s20=362371, s21=30380612, s22=30384103, s23=395865, s24=689807, s25=30296190] (stddev=104514490.49, mean=61848216.24, sum=1546205406)
-cpu#1: thrash_pct: [s1=8%, s2=28%, s3=8%, s4=4%, s5=3%, s6=2%, s7=0%, s8=1%, s9=0%, s10=2%, s11=0%, s12=1%, s13=0%, s14=1%, s15=0%, s16=1%, s17=1%, s18=3%, s19=0%, s20=0%, s21=2%, s22=1%, s23=0%, s24=0%, s25=2%]  (sum=69%)
-cpu_util#1: last:  [s1=0.05, s2=0.05, s3=0.03, s4=0.01, s5=0.01, s6=0.00, s7=0.00, s8=0.00, s9=0.00, s10=0.00, s11=0.00, s12=0.00, s13=0.00, s14=0.00, s15=0.00, s16=0.00, s17=0.00, s18=0.01, s19=0.00, s20=0.00, s21=0.00, s22=0.00, s23=0.00, s24=0.00, s25=0.00] (stddev=0.01, mean=0.01, sum=0)
-cpu_util#1: thrash_pct: [s1=8%, s2=28%, s3=8%, s4=4%, s5=3%, s6=2%, s7=0%, s8=1%, s9=0%, s10=2%, s11=0%, s12=1%, s13=0%, s14=1%, s15=0%, s16=1%, s17=1%, s18=3%, s19=0%, s20=0%, s21=2%, s22=1%, s23=0%, s24=0%, s25=2%]  (sum=69%)
+cpu#1: last:  [s1=4206946884, s2=4192160557, s3=6308915242, s4=4200098221, s5=4190855179, s6=4199054011, s7=4198384675, s8=4195969774, s9=4205041525, s10=4208294179, s11=4206362381, s12=4205362972, s13=4205781885, s14=4205720756, s15=4189319202, s16=4199761586, s17=4193214179, s18=4204100616, s19=4208198617, s20=4204672847, s21=4208825515, s22=2101879103, s23=4195299559, s24=4202456685, s25=4199007830] (stddev=594992424.39, mean=4201427359.20, sum=105035683980)
+cpu#1: thrash_pct: [s1=6%, s2=40%, s3=51%, s4=51%, s5=26%, s6=3%, s7=2%, s8=3%, s9=3%, s10=2%, s11=3%, s12=3%, s13=3%, s14=2%, s15=2%, s16=19%, s17=3%, s18=3%, s19=3%, s20=3%, s21=19%, s22=1%, s23=3%, s24=2%, s25=11%]  (sum=267%)
+cpu_util#1: last:  [s1=0.53, s2=0.52, s3=0.79, s4=0.53, s5=0.52, s6=0.52, s7=0.52, s8=0.52, s9=0.53, s10=0.53, s11=0.53, s12=0.53, s13=0.53, s14=0.53, s15=0.52, s16=0.52, s17=0.52, s18=0.53, s19=0.53, s20=0.53, s21=0.53, s22=0.26, s23=0.52, s24=0.53, s25=0.52] (stddev=0.07, mean=0.53, sum=13)
+cpu_util#1: thrash_pct: [s1=6%, s2=40%, s3=51%, s4=51%, s5=26%, s6=3%, s7=2%, s8=3%, s9=3%, s10=2%, s11=3%, s12=3%, s13=3%, s14=2%, s15=2%, s16=19%, s17=3%, s18=3%, s19=3%, s20=3%, s21=19%, s22=1%, s23=3%, s24=2%, s25=11%]  (sum=267%)
 leases#1: first: [s1=28, s2=14, s3=4, s4=3, s5=3, s6=2, s7=2, s8=2, s9=2, s10=2, s11=3, s12=2, s13=3, s14=3, s15=2, s16=3, s17=3, s18=3, s19=2, s20=2, s21=2, s22=3, s23=3, s24=3, s25=1] (stddev=5.43, mean=4.00, sum=100)
-leases#1: last:  [s1=13, s2=14, s3=10, s4=5, s5=5, s6=3, s7=2, s8=2, s9=2, s10=3, s11=3, s12=2, s13=3, s14=3, s15=2, s16=3, s17=3, s18=4, s19=2, s20=2, s21=3, s22=3, s23=3, s24=3, s25=2] (stddev=3.24, mean=4.00, sum=100)
-leases#1: thrash_pct: [s1=0%, s2=22%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%, s11=0%, s12=0%, s13=0%, s14=0%, s15=0%, s16=0%, s17=0%, s18=0%, s19=0%, s20=0%, s21=0%, s22=0%, s23=0%, s24=0%, s25=0%]  (sum=22%)
+leases#1: last:  [s1=3, s2=4, s3=5, s4=4, s5=4, s6=4, s7=4, s8=3, s9=4, s10=4, s11=5, s12=3, s13=5, s14=4, s15=4, s16=4, s17=4, s18=4, s19=4, s20=4, s21=4, s22=3, s23=5, s24=5, s25=3] (stddev=0.63, mean=4.00, sum=100)
+leases#1: thrash_pct: [s1=0%, s2=34%, s3=45%, s4=45%, s5=23%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%, s11=0%, s12=0%, s13=0%, s14=0%, s15=0%, s16=15%, s17=0%, s18=0%, s19=0%, s20=0%, s21=16%, s22=0%, s23=0%, s24=0%, s25=8%]  (sum=187%)
 replicas#1: first: [s1=56, s2=44, s3=25, s4=16, s5=12, s6=9, s7=8, s8=7, s9=7, s10=7, s11=7, s12=7, s13=7, s14=8, s15=7, s16=8, s17=7, s18=8, s19=7, s20=7, s21=7, s22=7, s23=7, s24=8, s25=7] (stddev=11.97, mean=12.00, sum=300)
-replicas#1: last:  [s1=56, s2=44, s3=25, s4=16, s5=12, s6=9, s7=8, s8=7, s9=7, s10=7, s11=7, s12=7, s13=7, s14=8, s15=7, s16=8, s17=7, s18=8, s19=7, s20=7, s21=7, s22=7, s23=7, s24=8, s25=7] (stddev=11.97, mean=12.00, sum=300)
-replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%, s11=0%, s12=0%, s13=0%, s14=0%, s15=0%, s16=0%, s17=0%, s18=0%, s19=0%, s20=0%, s21=0%, s22=0%, s23=0%, s24=0%, s25=0%]  (sum=0%)
-write_bytes_per_second#1: last:  [s1=9754, s2=9566, s3=9255, s4=9151, s5=9080, s6=9029, s7=9004, s8=9012, s9=9035, s10=9019, s11=8997, s12=8966, s13=9027, s14=9043, s15=9056, s16=9054, s17=9014, s18=9030, s19=9005, s20=8990, s21=9016, s22=9020, s23=9024, s24=9054, s25=9035] (stddev=179.12, mean=9089.44, sum=227236)
-write_bytes_per_second#1: thrash_pct: [s1=403%, s2=402%, s3=380%, s4=395%, s5=421%, s6=394%, s7=390%, s8=367%, s9=352%, s10=347%, s11=345%, s12=376%, s13=373%, s14=377%, s15=356%, s16=361%, s17=396%, s18=390%, s19=371%, s20=365%, s21=412%, s22=362%, s23=435%, s24=387%, s25=359%]  (sum=9516%)
-artifacts[mma-only]: 231ce55ad1d74827
+replicas#1: last:  [s1=54, s2=33, s3=19, s4=11, s5=10, s6=9, s7=10, s8=8, s9=9, s10=9, s11=8, s12=8, s13=9, s14=9, s15=9, s16=9, s17=8, s18=8, s19=9, s20=9, s21=8, s22=7, s23=9, s24=10, s25=8] (stddev=9.99, mean=12.00, sum=300)
+replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=5%, s5=5%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%, s11=0%, s12=0%, s13=0%, s14=0%, s15=0%, s16=9%, s17=0%, s18=0%, s19=0%, s20=0%, s21=9%, s22=0%, s23=0%, s24=0%, s25=4%]  (sum=31%)
+write_bytes_per_second#1: last:  [s1=9725, s2=9402, s3=9165, s4=9076, s5=9050, s6=9029, s7=9035, s8=9027, s9=9065, s10=9049, s11=9012, s12=8981, s13=9057, s14=9058, s15=9086, s16=9069, s17=9029, s18=9030, s19=9035, s20=9020, s21=9031, s22=9020, s23=9054, s24=9084, s25=9051] (stddev=150.73, mean=9089.60, sum=227240)
+write_bytes_per_second#1: thrash_pct: [s1=476%, s2=467%, s3=415%, s4=409%, s5=427%, s6=396%, s7=394%, s8=367%, s9=352%, s10=351%, s11=346%, s12=376%, s13=374%, s14=380%, s15=358%, s16=365%, s17=398%, s18=390%, s19=372%, s20=370%, s21=416%, s22=362%, s23=440%, s24=396%, s25=364%]  (sum=9762%)
+artifacts[mma-only]: 6abd02a180a2723e
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma_high_cpu_unable_to_shed_leases.txt

@@ -36,24 +36,24 @@ gen_ranges ranges=25 min_key=0 max_key=10000 placement_type=replica_placement
 {s1,s2,s4:*}:6
 {s1,s3:*,s5}:6
 
-gen_load rate=1000 rw_ratio=0 min_key=0 max_key=10000 raft_cpu_per_write=500000
+gen_load rate=5000 rw_ratio=0 min_key=0 max_key=10000 raft_cpu_per_write=1000000
 ----
 
 eval duration=30m samples=1 seed=42 cfgs=(mma-only) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
-cpu#1: last:  [s1=260292000, s2=300156000, s3=320067000, s4=299992333, s5=319992666] (stddev=21813256.71, mean=300099999.80, sum=1500499999)
-cpu#1: thrash_pct: [s1=12%, s2=23%, s3=23%, s4=24%, s5=25%]  (sum=108%)
-cpu_util#1: last:  [s1=0.03, s2=0.03, s3=0.04, s4=0.03, s5=0.04] (stddev=0.00, mean=0.03, sum=0)
-cpu_util#1: thrash_pct: [s1=12%, s2=23%, s3=23%, s4=24%, s5=25%]  (sum=108%)
+cpu#1: last:  [s1=2600894000, s2=3001220000, s3=3199993333, s4=3002060000, s5=3200832666] (stddev=218914332.20, mean=3000999999.80, sum=15004999999)
+cpu#1: thrash_pct: [s1=6%, s2=11%, s3=11%, s4=12%, s5=11%]  (sum=51%)
+cpu_util#1: last:  [s1=0.29, s2=0.33, s3=0.36, s4=0.33, s5=0.36] (stddev=0.02, mean=0.33, sum=2)
+cpu_util#1: thrash_pct: [s1=6%, s2=11%, s3=11%, s4=12%, s5=11%]  (sum=51%)
 leases#1: first: [s1=0, s2=7, s3=6, s4=6, s5=6] (stddev=2.53, mean=5.00, sum=25)
 leases#1: last:  [s1=0, s2=7, s3=6, s4=6, s5=6] (stddev=2.53, mean=5.00, sum=25)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
 replicas#1: first: [s1=25, s2=13, s3=13, s4=12, s5=12] (stddev=5.02, mean=15.00, sum=75)
 replicas#1: last:  [s1=13, s2=15, s3=16, s4=15, s5=16] (stddev=1.10, mean=15.00, sum=75)
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
-write_bytes_per_second#1: last:  [s1=520, s2=600, s3=640, s4=599, s5=639] (stddev=43.64, mean=599.60, sum=2998)
-write_bytes_per_second#1: thrash_pct: [s1=45%, s2=21%, s3=24%, s4=26%, s5=24%]  (sum=139%)
-artifacts[mma-only]: e970081fb35d1d29
+write_bytes_per_second#1: last:  [s1=2600, s2=3001, s3=3199, s4=3002, s5=3200] (stddev=218.91, mean=3000.40, sum=15002)
+write_bytes_per_second#1: thrash_pct: [s1=14%, s2=10%, s3=11%, s4=13%, s5=12%]  (sum=60%)
+artifacts[mma-only]: 3334492f2bf1a359
 ==========================
 
 # TODO(kvoli): Case (2)