Merge cockroachdb#160458

160458: asim: improve and CI-enable skewed_cpu_skewed_write r=tbg a=tbg

Add cold ranges to get it to parity with cockroachdb#153862.

Closes cockroachdb#153862.
Epic: CRDB-55052.

Co-authored-by: Tobias Grieger <tobias.b.grieger@gmail.com>
Co-authored-by: wenyihu6 <wenyi@cockroachlabs.com>

Author: 3 people

pkg/kv/kvserver/asim/tests/datadriven_simulation_test.go

@@ -634,6 +634,15 @@ func TestDataDriven(t *testing.T) {
 					// the first LBRebalancingMode configuration since this string is only
 					// generated for once at the start.
 					var stateStrForOnce string
+					// TODO(tbg): there's tension between having subtests per
+					// config (for filtering and individual failure reporting),
+					// running them concurrently (for speed), and combining
+					// their output into a single testdata file.  Parallel
+					// subtests don't start until the parent yields, which
+					// deadlocks if we block waiting for results to combine. A
+					// clean-ish fix is to split each config into its own
+					// testdata file, sharing common setup via an `include`
+					// directive (see TestClusterState for an example).
 					for _, mv := range cfgs {
 						t.Run(mv, func(t *testing.T) {
 							ctx := logtags.AddTag(context.Background(), "name", name+"/"+mv)
@@ -694,6 +703,7 @@ func TestDataDriven(t *testing.T) {
 
 							runs = append(runs, run)
 
+							_, _ = fmt.Fprintf(&buf, "%s:\n", mv)
 							// Generate artifacts. Hash artifact input data to ensure they are
 							// up to date.
 							hasher := fnv.New64a()
@@ -706,7 +716,7 @@ func TestDataDriven(t *testing.T) {
 
 							// For each sample that had at least one failing assertion,
 							// report the sample and every failing assertion.
-							_, _ = fmt.Fprintf(&buf, "artifacts[%s]: %x\n", mv, artifactsHash)
+							_, _ = fmt.Fprintf(&buf, "hash: %x\n", artifactsHash)
 							for sample, failString := range sampleAssertFailures {
 								if failString != "" {
 									_, _ = fmt.Fprintf(&buf, "failed assertion sample %d\n%s\n",

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/constraint_satisfaction_full_disk.txt

@@ -38,6 +38,7 @@ setting split_queue_enabled=false
 
 eval duration=3m samples=1 seed=42 cfgs=(sma-count,mma-count) metrics=(cpu,cpu_util,leases,replicas,disk_fraction_used)
 ----
+sma-count:
 disk_fraction_used#1: first: [s1=0.61, s2=0.61, s3=0.00, s4=0.61, s5=0.61, s6=0.61, s7=0.00, s8=0.00, s9=0.00] (stddev=0.30, mean=0.34, sum=3)
 disk_fraction_used#1: last:  [s1=0.49, s2=0.37, s3=0.37, s4=0.61, s5=0.61, s6=0.61, s7=0.00, s8=0.00, s9=0.00] (stddev=0.26, mean=0.34, sum=3)
 disk_fraction_used#1: thrash_pct: [s1=48%, s2=26%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=74%)
@@ -47,8 +48,9 @@ leases#1: thrash_pct: [s1=24%, s2=21%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%,
 replicas#1: first: [s1=10, s2=10, s3=0, s4=10, s5=10, s6=10, s7=0, s8=0, s9=0] (stddev=4.97, mean=5.56, sum=50)
 replicas#1: last:  [s1=8, s2=6, s3=6, s4=10, s5=10, s6=10, s7=0, s8=0, s9=0] (stddev=4.19, mean=5.56, sum=50)
 replicas#1: thrash_pct: [s1=48%, s2=26%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=74%)
-artifacts[sma-count]: 7ef09084dfb9e631
+hash: 7ef09084dfb9e631
 ==========================
+mma-count:
 disk_fraction_used#1: first: [s1=0.61, s2=0.61, s3=0.00, s4=0.61, s5=0.61, s6=0.61, s7=0.00, s8=0.00, s9=0.00] (stddev=0.30, mean=0.34, sum=3)
 disk_fraction_used#1: last:  [s1=0.43, s2=0.37, s3=0.43, s4=0.61, s5=0.61, s6=0.61, s7=0.00, s8=0.00, s9=0.00] (stddev=0.25, mean=0.34, sum=3)
 disk_fraction_used#1: thrash_pct: [s1=72%, s2=50%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=122%)
@@ -58,5 +60,5 @@ leases#1: thrash_pct: [s1=24%, s2=21%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%,
 replicas#1: first: [s1=10, s2=10, s3=0, s4=10, s5=10, s6=10, s7=0, s8=0, s9=0] (stddev=4.97, mean=5.56, sum=50)
 replicas#1: last:  [s1=7, s2=6, s3=7, s4=10, s5=10, s6=10, s7=0, s8=0, s9=0] (stddev=4.17, mean=5.56, sum=50)
 replicas#1: thrash_pct: [s1=72%, s2=50%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=122%)
-artifacts[mma-count]: 7a19c650f32fe6f4
+hash: 7a19c650f32fe6f4
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/full_disk.txt

@@ -21,6 +21,7 @@ gen_ranges ranges=15 bytes_mib=500 repl_factor=3 placement_type=skewed
 
 eval duration=10m seed=42 cfgs=(mma-only) metrics=(disk_fraction_used,replicas,leases,cpu,cpu_util,mma-count)
 ----
+mma-only:
 disk_fraction_used#1: first: [s1=0.92, s2=0.85, s3=0.49, s4=0.24, s5=0.24] (stddev=0.29, mean=0.55, sum=3)
 disk_fraction_used#1: last:  [s1=0.55, s2=0.55, s3=0.55, s4=0.55, s5=0.55] (stddev=0.00, mean=0.55, sum=3)
 disk_fraction_used#1: thrash_pct: [s1=23%, s2=116%, s3=111%, s4=0%, s5=22%]  (sum=272%)
@@ -30,5 +31,5 @@ leases#1: thrash_pct: [s1=0%, s2=63%, s3=60%, s4=0%, s5=0%]  (sum=122%)
 replicas#1: first: [s1=15, s2=14, s3=8, s4=4, s5=4] (stddev=4.73, mean=9.00, sum=45)
 replicas#1: last:  [s1=9, s2=9, s3=9, s4=9, s5=9] (stddev=0.00, mean=9.00, sum=45)
 replicas#1: thrash_pct: [s1=23%, s2=116%, s3=111%, s4=0%, s5=22%]  (sum=272%)
-artifacts[mma-only]: a9369e989cbbae6d
+hash: a9369e989cbbae6d
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/heterogeneous_cpu.txt

@@ -32,11 +32,12 @@ asserting: max_{stores}(cpu_util)/mean_{stores}(cpu_util) ≤ 1.10 at each of la
 
 eval cfgs=(sma-count,mma-count) duration=3m metrics=(cpu,cpu_util)
 ----
+sma-count:
 cpu#1: last:  [s1=6702050000, s2=6694069444, s3=6603880555] (stddev=44515870.96, mean=6666666666.33, sum=19999999999)
 cpu#1: thrash_pct: [s1=133%, s2=123%, s3=139%]  (sum=395%)
 cpu_util#1: last:  [s1=0.84, s2=0.84, s3=0.41] (stddev=0.20, mean=0.70, sum=2)
 cpu_util#1: thrash_pct: [s1=5%, s2=5%, s3=3%]  (sum=12%)
-artifacts[sma-count]: 1306b93e8b8d4a39
+hash: 1306b93e8b8d4a39
 failed assertion sample 1
   balance stat=cpu_util threshold=(≤1.10) ticks=6
 	max/mean=1.20 tick=0
@@ -46,11 +47,12 @@ failed assertion sample 1
 	max/mean=1.20 tick=4
 	max/mean=1.20 tick=5
 ==========================
+mma-count:
 cpu#1: last:  [s1=6399652777, s2=6391952777, s3=7205401827] (stddev=381661598.00, mean=6665669127.00, sum=19997007381)
 cpu#1: thrash_pct: [s1=24%, s2=29%, s3=31%]  (sum=84%)
 cpu_util#1: last:  [s1=0.80, s2=0.80, s3=0.45] (stddev=0.16, mean=0.68, sum=2)
 cpu_util#1: thrash_pct: [s1=6%, s2=7%, s3=4%]  (sum=16%)
-artifacts[mma-count]: c51384f72bf7101d
+hash: c51384f72bf7101d
 failed assertion sample 1
   balance stat=cpu_util threshold=(≤1.10) ticks=6
 	max/mean=1.17 tick=0

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu.txt

@@ -62,6 +62,7 @@ asserting: |leases(t)/mean_{T}(leases) - 1| ≤ 0.10 ∀ t∈T and each store (T
 
 eval duration=20m samples=1 seed=42 cfgs=(mma-count,mma-only) metrics=(cpu,cpu_util,replicas,leases)
 ----
+mma-count:
 cpu#1: last:  [s1=1541887030, s2=1713235417, s3=1580118619, s4=1618283496, s5=1563312037, s6=1657951235, s7=1307364950, s8=1552531971, s9=1561935680, s10=1645749580] (stddev=103163155.55, mean=1574237001.50, sum=15742370015)
 cpu#1: thrash_pct: [s1=25%, s2=29%, s3=18%, s4=29%, s5=28%, s6=16%, s7=19%, s8=27%, s9=30%, s10=18%]  (sum=238%)
 cpu_util#1: last:  [s1=0.19, s2=0.21, s3=0.20, s4=0.20, s5=0.20, s6=0.21, s7=0.16, s8=0.19, s9=0.20, s10=0.21] (stddev=0.01, mean=0.20, sum=2)
@@ -72,8 +73,9 @@ leases#1: thrash_pct: [s1=0%, s2=9%, s3=8%, s4=9%, s5=9%, s6=0%, s7=0%, s8=9%, s
 replicas#1: first: [s1=80, s2=70, s3=51, s4=42, s5=37, s6=35, s7=34, s8=33, s9=34, s10=34] (stddev=16.02, mean=45.00, sum=450)
 replicas#1: last:  [s1=45, s2=44, s3=45, s4=46, s5=43, s6=45, s7=47, s8=45, s9=45, s10=45] (stddev=1.00, mean=45.00, sum=450)
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=5%, s6=0%, s7=0%, s8=0%, s9=6%, s10=0%]  (sum=11%)
-artifacts[mma-count]: 2e50737f44fc4950
+hash: 2e50737f44fc4950
 ==========================
+mma-only:
 cpu#1: last:  [s1=1706498399, s2=1615687783, s3=1677746697, s4=1624652238, s5=1648924305, s6=1640076380, s7=1135947134, s8=1438619020, s9=1699568316, s10=1553238061] (stddev=164018033.83, mean=1574095833.30, sum=15740958333)
 cpu#1: thrash_pct: [s1=32%, s2=85%, s3=84%, s4=36%, s5=17%, s6=37%, s7=15%, s8=16%, s9=22%, s10=20%]  (sum=364%)
 cpu_util#1: last:  [s1=0.21, s2=0.20, s3=0.21, s4=0.20, s5=0.21, s6=0.21, s7=0.14, s8=0.18, s9=0.21, s10=0.19] (stddev=0.02, mean=0.20, sum=2)
@@ -84,5 +86,5 @@ leases#1: thrash_pct: [s1=0%, s2=41%, s3=66%, s4=37%, s5=0%, s6=39%, s7=0%, s8=0
 replicas#1: first: [s1=80, s2=70, s3=51, s4=42, s5=37, s6=35, s7=34, s8=33, s9=34, s10=34] (stddev=16.02, mean=45.00, sum=450)
 replicas#1: last:  [s1=80, s2=58, s3=46, s4=45, s5=38, s6=37, s7=35, s8=36, s9=37, s10=38] (stddev=13.42, mean=45.00, sum=450)
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=5%, s4=0%, s5=0%, s6=5%, s7=0%, s8=0%, s9=0%, s10=0%]  (sum=10%)
-artifacts[mma-only]: 7478cadb8f2e4145
+hash: 7478cadb8f2e4145
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_25nodes.txt

@@ -25,6 +25,7 @@ gen_load rate=15000 rw_ratio=0.95 min_block=1 max_block=1 request_cpu_per_access
 
 eval duration=25m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
+mma-only:
 cpu#1: last:  [s1=463047636, s2=464742044, s3=465584574, s4=464309298, s5=465847383, s6=466444044, s7=463910580, s8=466175498, s9=464760769, s10=467967426, s11=468395602, s12=468693867, s13=468326290, s14=319992397, s15=468057733, s16=403289930, s17=467566175, s18=340531137, s19=467968120, s20=468090063, s21=361926413, s22=404937284, s23=469792185, s24=383090767, s25=319789535] (stddev=50329198.14, mean=437329470.00, sum=10933236750)
 cpu#1: thrash_pct: [s1=13%, s2=39%, s3=29%, s4=12%, s5=41%, s6=29%, s7=13%, s8=29%, s9=38%, s10=8%, s11=7%, s12=7%, s13=10%, s14=6%, s15=8%, s16=7%, s17=12%, s18=6%, s19=7%, s20=8%, s21=6%, s22=7%, s23=9%, s24=6%, s25=6%]  (sum=362%)
 cpu_util#1: last:  [s1=0.06, s2=0.06, s3=0.06, s4=0.06, s5=0.06, s6=0.06, s7=0.06, s8=0.06, s9=0.06, s10=0.06, s11=0.06, s12=0.06, s13=0.06, s14=0.04, s15=0.06, s16=0.05, s17=0.06, s18=0.04, s19=0.06, s20=0.06, s21=0.05, s22=0.05, s23=0.06, s24=0.05, s25=0.04] (stddev=0.01, mean=0.05, sum=1)
@@ -37,8 +38,9 @@ replicas#1: last:  [s1=84, s2=153, s3=156, s4=86, s5=155, s6=157, s7=85, s8=155,
 replicas#1: thrash_pct: [s1=0%, s2=11%, s3=5%, s4=0%, s5=13%, s6=8%, s7=0%, s8=9%, s9=10%, s10=0%, s11=0%, s12=0%, s13=3%, s14=0%, s15=0%, s16=0%, s17=4%, s18=0%, s19=0%, s20=2%, s21=0%, s22=0%, s23=2%, s24=0%, s25=0%]  (sum=67%)
 write_bytes_per_second#1: last:  [s1=125, s2=229, s3=233, s4=128, s5=231, s6=235, s7=126, s8=232, s9=227, s10=32, s11=33, s12=33, s13=32, s14=22, s15=32, s16=28, s17=32, s18=23, s19=32, s20=32, s21=25, s22=28, s23=33, s24=26, s25=22] (stddev=85.61, mean=89.24, sum=2231)
 write_bytes_per_second#1: thrash_pct: [s1=179%, s2=262%, s3=281%, s4=185%, s5=247%, s6=266%, s7=180%, s8=250%, s9=247%, s10=29%, s11=23%, s12=8%, s13=31%, s14=24%, s15=22%, s16=10%, s17=27%, s18=23%, s19=34%, s20=22%, s21=12%, s22=19%, s23=33%, s24=58%, s25=21%]  (sum=2492%)
-artifacts[mma-only]: 99cd3e4d17a5a700
+hash: 99cd3e4d17a5a700
 ==========================
+mma-count:
 cpu#1: last:  [s1=479911909, s2=373580982, s3=417004825, s4=481191026, s5=437968812, s6=418529844, s7=480521667, s8=416565076, s9=375203500, s10=416487421, s11=459073056, s12=417390139, s13=458452447, s14=438198669, s15=438626986, s16=436488614, s17=459362930, s18=437947945, s19=458740016, s20=438620980, s21=437601149, s22=458747503, s23=416674300, s24=436424336, s25=460194196] (stddev=27439196.31, mean=437980333.12, sum=10949508328)
 cpu#1: thrash_pct: [s1=7%, s2=8%, s3=7%, s4=8%, s5=8%, s6=7%, s7=8%, s8=7%, s9=7%, s10=13%, s11=7%, s12=7%, s13=10%, s14=5%, s15=9%, s16=12%, s17=6%, s18=6%, s19=10%, s20=8%, s21=10%, s22=7%, s23=7%, s24=7%, s25=6%]  (sum=199%)
 cpu_util#1: last:  [s1=0.06, s2=0.05, s3=0.05, s4=0.06, s5=0.05, s6=0.05, s7=0.06, s8=0.05, s9=0.05, s10=0.05, s11=0.06, s12=0.05, s13=0.06, s14=0.05, s15=0.05, s16=0.05, s17=0.06, s18=0.05, s19=0.06, s20=0.05, s21=0.05, s22=0.06, s23=0.05, s24=0.05, s25=0.06] (stddev=0.00, mean=0.05, sum=1)
@@ -51,5 +53,5 @@ replicas#1: last:  [s1=61, s2=60, s3=61, s4=63, s5=58, s6=63, s7=63, s8=60, s9=6
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%, s11=0%, s12=0%, s13=0%, s14=0%, s15=5%, s16=0%, s17=0%, s18=0%, s19=0%, s20=0%, s21=0%, s22=0%, s23=0%, s24=0%, s25=2%]  (sum=7%)
 write_bytes_per_second#1: last:  [s1=91, s2=89, s3=91, s4=94, s5=87, s6=94, s7=94, s8=89, s9=91, s10=87, s11=86, s12=88, s13=87, s14=91, s15=89, s16=85, s17=92, s18=91, s19=88, s20=90, s21=87, s22=90, s23=88, s24=87, s25=92] (stddev=2.50, mean=89.52, sum=2238)
 write_bytes_per_second#1: thrash_pct: [s1=74%, s2=98%, s3=89%, s4=68%, s5=111%, s6=106%, s7=63%, s8=104%, s9=110%, s10=45%, s11=38%, s12=48%, s13=34%, s14=33%, s15=81%, s16=44%, s17=47%, s18=43%, s19=32%, s20=68%, s21=37%, s22=67%, s23=40%, s24=70%, s25=55%]  (sum=1606%)
-artifacts[mma-count]: 783f63043b18076a
+hash: 783f63043b18076a
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_able_to_shed_leases.txt

@@ -27,6 +27,7 @@ gen_load rate=50000 rw_ratio=0 min_key=0 max_key=10000 raft_cpu_per_write=100000
 
 eval duration=5m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
+mma-only:
 cpu#1: last:  [s1=2598964872, s2=2599396812, s3=2996720523, s4=3199182403, s5=3599520277] (stddev=379582220.28, mean=2998756977.40, sum=14993784887)
 cpu#1: thrash_pct: [s1=161%, s2=82%, s3=144%, s4=176%, s5=158%]  (sum=721%)
 cpu_util#1: last:  [s1=0.29, s2=0.29, s3=0.33, s4=0.36, s5=0.40] (stddev=0.04, mean=0.33, sum=2)
@@ -39,8 +40,9 @@ replicas#1: last:  [s1=13, s2=13, s3=15, s4=16, s5=18] (stddev=1.90, mean=15.00,
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
 write_bytes_per_second#1: last:  [s1=25989, s2=25993, s3=29967, s4=31991, s5=35995] (stddev=3796.01, mean=29987.00, sum=149935)
 write_bytes_per_second#1: thrash_pct: [s1=161%, s2=82%, s3=144%, s4=176%, s5=158%]  (sum=722%)
-artifacts[mma-only]: 9a1122b07e8a1e71
+hash: 9a1122b07e8a1e71
 ==========================
+mma-count:
 cpu#1: last:  [s1=3200015723, s2=2799409624, s3=3199923468, s4=2800455801, s5=2999667444] (stddev=178902277.04, mean=2999894412.00, sum=14999472060)
 cpu#1: thrash_pct: [s1=109%, s2=62%, s3=49%, s4=108%, s5=79%]  (sum=408%)
 cpu_util#1: last:  [s1=0.36, s2=0.31, s3=0.36, s4=0.31, s5=0.33] (stddev=0.02, mean=0.33, sum=2)
@@ -53,5 +55,5 @@ replicas#1: last:  [s1=16, s2=14, s3=16, s4=14, s5=15] (stddev=0.89, mean=15.00,
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
 write_bytes_per_second#1: last:  [s1=32000, s2=27994, s3=31999, s4=28004, s5=29996] (stddev=1789.08, mean=29998.60, sum=149993)
 write_bytes_per_second#1: thrash_pct: [s1=109%, s2=62%, s3=49%, s4=107%, s5=79%]  (sum=408%)
-artifacts[mma-count]: afcb108e54d06e3c
+hash: afcb108e54d06e3c
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_unable_to_shed_leases.txt

@@ -39,6 +39,7 @@ gen_load rate=5000 rw_ratio=0 min_key=0 max_key=10000 raft_cpu_per_write=1000000
 
 eval duration=5m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
+mma-only:
 cpu#1: last:  [s1=2594640000, s2=2997280000, s3=3203736666, s4=3000476666, s5=3203866666] (stddev=222438576.60, mean=2999999999.60, sum=14999999998)
 cpu#1: thrash_pct: [s1=2%, s2=8%, s3=8%, s4=9%, s5=8%]  (sum=35%)
 cpu_util#1: last:  [s1=0.29, s2=0.33, s3=0.36, s4=0.33, s5=0.36] (stddev=0.02, mean=0.33, sum=2)
@@ -51,8 +52,9 @@ replicas#1: last:  [s1=13, s2=15, s3=16, s4=15, s5=16] (stddev=1.10, mean=15.00,
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
 write_bytes_per_second#1: last:  [s1=2594, s2=2997, s3=3203, s4=3000, s5=3203] (stddev=222.38, mean=2999.40, sum=14997)
 write_bytes_per_second#1: thrash_pct: [s1=10%, s2=8%, s3=8%, s4=8%, s5=8%]  (sum=42%)
-artifacts[mma-only]: 40790b28b0714c13
+hash: 40790b28b0714c13
 ==========================
+mma-count:
 cpu#1: last:  [s1=2602423333, s2=3197976666, s3=3199683333, s4=2997496666, s5=3002420000] (stddev=217783159.58, mean=2999999999.60, sum=14999999998)
 cpu#1: thrash_pct: [s1=17%, s2=17%, s3=16%, s4=13%, s5=13%]  (sum=77%)
 cpu_util#1: last:  [s1=0.29, s2=0.36, s3=0.36, s4=0.33, s5=0.33] (stddev=0.02, mean=0.33, sum=2)
@@ -65,5 +67,5 @@ replicas#1: last:  [s1=13, s2=16, s3=16, s4=15, s5=15] (stddev=1.10, mean=15.00,
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
 write_bytes_per_second#1: last:  [s1=2602, s2=3197, s3=3199, s4=2997, s5=3002] (stddev=217.64, mean=2999.40, sum=14997)
 write_bytes_per_second#1: thrash_pct: [s1=17%, s2=16%, s3=15%, s4=12%, s5=14%]  (sum=74%)
-artifacts[mma-count]: 7b308e57d4c5870e
+hash: 7b308e57d4c5870e
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_write_uniform_cpu.txt

@@ -31,6 +31,7 @@ setting split_queue_enabled=false
 
 eval duration=20m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(disk_fraction_used,cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
+mma-only:
 cpu#1: last:  [s1=166733333, s2=333087500, s3=332200000, s4=166087500, s5=165008333, s6=332937500, s7=166958333, s8=167408333, s9=334327114, s10=334445833, s11=333762299, s12=166275000, s13=166883333, s14=165841666, s15=167395833, s16=332033333, s17=337815597, s18=332821681, s19=166062500, s20=332395833] (stddev=83567992.93, mean=250024042.70, sum=5000480854)
 cpu#1: thrash_pct: [s1=44%, s2=69%, s3=60%, s4=48%, s5=45%, s6=56%, s7=40%, s8=61%, s9=79%, s10=61%, s11=158%, s12=43%, s13=45%, s14=63%, s15=43%, s16=158%, s17=176%, s18=78%, s19=53%, s20=59%]  (sum=1439%)
 cpu_util#1: last:  [s1=0.17, s2=0.17, s3=0.17, s4=0.17, s5=0.17, s6=0.17, s7=0.11, s8=0.11, s9=0.22, s10=0.22, s11=0.17, s12=0.17, s13=0.11, s14=0.11, s15=0.17, s16=0.17, s17=0.22, s18=0.22, s19=0.17, s20=0.17] (stddev=0.04, mean=0.17, sum=3)
@@ -46,8 +47,9 @@ replicas#1: last:  [s1=8, s2=18, s3=17, s4=8, s5=7, s6=9, s7=6, s8=8, s9=10, s10
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=21%, s5=13%, s6=0%, s7=40%, s8=7%, s9=8%, s10=0%, s11=8%, s12=0%, s13=8%, s14=7%, s15=0%, s16=0%, s17=7%, s18=0%, s19=28%, s20=0%]  (sum=147%)
 write_bytes_per_second#1: last:  [s1=2664475, s2=9990961, s3=8656089, s4=2664418, s5=2666355, s6=2663658, s7=666268, s8=2682724, s9=3332509, s10=3327703, s11=2664669, s12=2663382, s13=2682644, s14=665509, s15=2664546, s16=665509, s17=1997665, s18=3349273, s19=2662437, s20=666745] (stddev=2292409.54, mean=2999876.95, sum=59997539)
 write_bytes_per_second#1: thrash_pct: [s1=29%, s2=53%, s3=57%, s4=37%, s5=9%, s6=16%, s7=41%, s8=10%, s9=9%, s10=10%, s11=2%, s12=2%, s13=2%, s14=1%, s15=2%, s16=1%, s17=9%, s18=2%, s19=30%, s20=15%]  (sum=338%)
-artifacts[mma-only]: b3d73568e2e3aedb
+hash: b3d73568e2e3aedb
 ==========================
+mma-count:
 cpu#1: last:  [s1=333742344, s2=164084967, s3=166947040, s4=166609311, s5=164748455, s6=332784476, s7=335351639, s8=334494907, s9=336447928, s10=166739562, s11=334841106, s12=332918112, s13=332816550, s14=164598425, s15=334779339, s16=167056204, s17=0, s18=169782608, s19=330150330, s20=166952104] (stddev=98589571.98, mean=241792270.35, sum=4835845407)
 cpu#1: thrash_pct: [s1=326%, s2=717%, s3=405%, s4=376%, s5=379%, s6=356%, s7=467%, s8=911%, s9=596%, s10=316%, s11=351%, s12=311%, s13=285%, s14=486%, s15=319%, s16=190%, s17=388%, s18=593%, s19=592%, s20=371%]  (sum=8735%)
 cpu_util#1: last:  [s1=0.17, s2=0.17, s3=0.11, s4=0.11, s5=0.17, s6=0.17, s7=0.22, s8=0.22, s9=0.17, s10=0.17, s11=0.22, s12=0.22, s13=0.17, s14=0.17, s15=0.17, s16=0.17, s17=0.06, s18=0.06, s19=0.17, s20=0.17] (stddev=0.05, mean=0.16, sum=3)
@@ -63,5 +65,5 @@ replicas#1: last:  [s1=9, s2=9, s3=9, s4=7, s5=8, s6=9, s7=10, s8=8, s9=8, s10=9
 replicas#1: thrash_pct: [s1=297%, s2=405%, s3=262%, s4=421%, s5=239%, s6=356%, s7=196%, s8=730%, s9=595%, s10=183%, s11=301%, s12=209%, s13=124%, s14=222%, s15=190%, s16=306%, s17=216%, s18=358%, s19=447%, s20=401%]  (sum=6457%)
 write_bytes_per_second#1: last:  [s1=2663008, s2=3321052, s3=3330664, s4=1988212, s5=2659475, s6=2689448, s7=1995561, s8=1999752, s9=2678213, s10=2666906, s11=6660568, s12=3326073, s13=2660766, s14=2663583, s15=4004468, s16=2667484, s17=4657240, s18=4675995, s19=2684404, s20=0] (stddev=1290248.94, mean=2999643.60, sum=59992872)
 write_bytes_per_second#1: thrash_pct: [s1=308%, s2=385%, s3=290%, s4=397%, s5=296%, s6=281%, s7=269%, s8=517%, s9=402%, s10=93%, s11=272%, s12=195%, s13=98%, s14=249%, s15=182%, s16=316%, s17=156%, s18=357%, s19=362%, s20=319%]  (sum=5742%)
-artifacts[mma-count]: 519e91692a65fe4d
+hash: 519e91692a65fe4d
 ==========================