asim: skip prefix of zeroes in thrashing computation

This works around unexpected variation being detected when various timeseries
first get populated.

Reviewing the datadriven diffs shows that most thrashing percentages go down.
A few have increased, since they previously included these zero values into
the computation of the range of the time series. Now that these are stripped,
the denominator in the conversion from trend-discounting total variation becomes
smaller, resulting in increasing tdtv percentages. These changes are positive,
since the variation was always there, but previously masked.

Author: tbg

pkg/kv/kvserver/asim/history/history.go

@@ -67,7 +67,22 @@ func (h *History) ThrashingForStat(stat string) ThrashingSlice {
 
 	ths := make(ThrashingSlice, numStores)
 	for storeIdx := range vsByStore {
-		th := computeThrashing(vsByStore[storeIdx])
+		sl := vsByStore[storeIdx]
+		// HACK: instead of the slice directly, we measure the thrashing of a slice
+		// that has all leading zeroes removed. This works around the fact that some
+		// timeseries only show sensible values after an initial period of
+		// inactivity. For example, CPU usage is zero until the first stats tick.
+		// Without this hack, the large initial jump from zero to the first value
+		// would be interpreted as variation.
+		noLeadingZeroes := sl
+		for i := 0; i < len(sl); i++ {
+			if sl[i] == 0 {
+				noLeadingZeroes = sl[i+1:]
+				continue
+			}
+			break
+		}
+		th := computeThrashing(noLeadingZeroes)
 		ths[storeIdx] = th
 	}
 	ths.normalize()

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_multi_store.txt

@@ -23,12 +23,12 @@ eval duration=5m seed=42 metrics=(leases) cfgs=(sma-count,mma-only)
 ----
 leases#1: first: [s1=8, s2=3, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=1, s10=0, s11=1, s12=0, s13=1, s14=0] (stddev=2.10, mean=1.00, sum=14)
 leases#1: last:  [s1=1, s2=1, s3=1, s4=1, s5=1, s6=1, s7=1, s8=1, s9=1, s10=1, s11=1, s12=1, s13=1, s14=1] (stddev=0.00, mean=1.00, sum=14)
-leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=27%, s9=25%, s10=27%, s11=25%, s12=0%, s13=50%, s14=27%]  (sum=181%)
+leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=25%, s9=25%, s10=0%, s11=25%, s12=0%, s13=50%, s14=25%]  (sum=150%)
 artifacts[sma-count]: 6322c40ff6c7b47c
 ==========================
 leases#1: first: [s1=8, s2=3, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=1, s10=0, s11=1, s12=0, s13=1, s14=0] (stddev=2.10, mean=1.00, sum=14)
 leases#1: last:  [s1=4, s2=1, s3=0, s4=0, s5=0, s6=1, s7=0, s8=0, s9=1, s10=1, s11=1, s12=1, s13=2, s14=2] (stddev=1.07, mean=1.00, sum=14)
-leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=25%, s9=0%, s10=0%, s11=25%, s12=27%, s13=0%, s14=0%]  (sum=77%)
+leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%, s11=25%, s12=0%, s13=0%, s14=0%]  (sum=25%)
 artifacts[mma-only]: ed1b3ca87bbb8148
 failed assertion sample 1
   balance stat=leases threshold=(≤1.00) ticks=6

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing.txt

@@ -42,19 +42,19 @@ asserting: |qps(t)/mean_{T}(qps) - 1| ≤ 0.05 ∀ t∈T and each store (T=last
 eval duration=3m samples=2 seed=42 metrics=(qps,replica_moves) cfgs=(sma-count,mma-only)
 ----
 qps#1: last:  [s1=998, s2=998, s3=1001, s4=995, s5=998, s6=1002, s7=1001] (stddev=2.27, mean=999.00, sum=6993)
-qps#1: thrash_pct: [s1=144%, s2=77%, s3=6%, s4=76%, s5=6%, s6=5%, s7=4%]  (sum=319%)
+qps#1: thrash_pct: [s1=13%, s2=9%, s3=5%, s4=8%, s5=5%, s6=5%, s7=4%]  (sum=48%)
 replica_moves#1: last:  [s1=2, s2=1, s3=0, s4=1, s5=1, s6=1, s7=0] (stddev=0.64, mean=0.86, sum=6)
 replica_moves#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%]  (sum=0%)
 qps#2: last:  [s1=996, s2=996, s3=1000, s4=998, s5=1001, s6=1003, s7=1000] (stddev=2.42, mean=999.14, sum=6994)
-qps#2: thrash_pct: [s1=143%, s2=75%, s3=5%, s4=5%, s5=5%, s6=76%, s7=5%]  (sum=314%)
+qps#2: thrash_pct: [s1=8%, s2=11%, s3=5%, s4=5%, s5=5%, s6=105%, s7=5%]  (sum=145%)
 replica_moves#2: last:  [s1=2, s2=1, s3=0, s4=0, s5=0, s6=1, s7=0] (stddev=0.73, mean=0.57, sum=4)
 replica_moves#2: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%]  (sum=0%)
 artifacts[sma-count]: 26755534702067c5
 ==========================
 qps#1: last:  [s1=3999, s2=1997, s3=0, s4=0, s5=0, s6=1003, s7=0] (stddev=1413.61, mean=999.86, sum=6999)
-qps#1: thrash_pct: [s1=8%, s2=5%, s3=0%, s4=0%, s5=0%, s6=4%, s7=0%]  (sum=17%)
+qps#1: thrash_pct: [s1=5%, s2=5%, s3=0%, s4=0%, s5=0%, s6=4%, s7=0%]  (sum=13%)
 qps#2: last:  [s1=3996, s2=1996, s3=0, s4=0, s5=0, s6=1006, s7=0] (stddev=1412.60, mean=999.71, sum=6998)
-qps#2: thrash_pct: [s1=7%, s2=4%, s3=0%, s4=0%, s5=0%, s6=5%, s7=0%]  (sum=16%)
+qps#2: thrash_pct: [s1=5%, s2=4%, s3=0%, s4=0%, s5=0%, s6=3%, s7=0%]  (sum=13%)
 artifacts[mma-only]: dad89302cc72743e
 failed assertion sample 1
   balance stat=qps threshold=(≤1.15) ticks=6
@@ -90,19 +90,19 @@ setting gossip_delay=20s
 eval duration=5m samples=2 seed=42 metrics=(qps,replica_moves) cfgs=(sma-count,mma-only)
 ----
 qps#1: last:  [s1=994, s2=1004, s3=1002, s4=1002, s5=1000, s6=997, s7=999] (stddev=3.15, mean=999.71, sum=6998)
-qps#1: thrash_pct: [s1=259%, s2=109%, s3=5%, s4=59%, s5=109%, s6=109%, s7=57%]  (sum=707%)
+qps#1: thrash_pct: [s1=114%, s2=58%, s3=3%, s4=4%, s5=104%, s6=105%, s7=53%]  (sum=443%)
 replica_moves#1: last:  [s1=6, s2=4, s3=0, s4=1, s5=3, s6=5, s7=2] (stddev=2.00, mean=3.00, sum=21)
 replica_moves#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%]  (sum=0%)
 qps#2: last:  [s1=999, s2=998, s3=998, s4=998, s5=1000, s6=999, s7=1003] (stddev=1.67, mean=999.29, sum=6995)
-qps#2: thrash_pct: [s1=510%, s2=259%, s3=58%, s4=59%, s5=361%, s6=262%, s7=363%]  (sum=1873%)
+qps#2: thrash_pct: [s1=368%, s2=210%, s3=54%, s4=54%, s5=355%, s6=258%, s7=358%]  (sum=1658%)
 replica_moves#2: last:  [s1=15, s2=13, s3=5, s4=3, s5=10, s6=9, s7=8] (stddev=3.89, mean=9.00, sum=63)
 replica_moves#2: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%]  (sum=0%)
 artifacts[sma-count]: d6940fa02dce5639
 ==========================
 qps#1: last:  [s1=3995, s2=1998, s3=0, s4=0, s5=0, s6=1005, s7=0] (stddev=1412.50, mean=999.71, sum=6998)
-qps#1: thrash_pct: [s1=8%, s2=6%, s3=0%, s4=0%, s5=0%, s6=4%, s7=0%]  (sum=19%)
+qps#1: thrash_pct: [s1=5%, s2=6%, s3=0%, s4=0%, s5=0%, s6=4%, s7=0%]  (sum=15%)
 qps#2: last:  [s1=4000, s2=1995, s3=0, s4=0, s5=0, s6=1004, s7=0] (stddev=1413.71, mean=999.86, sum=6999)
-qps#2: thrash_pct: [s1=7%, s2=5%, s3=0%, s4=0%, s5=0%, s6=5%, s7=0%]  (sum=17%)
+qps#2: thrash_pct: [s1=6%, s2=4%, s3=0%, s4=0%, s5=0%, s6=4%, s7=0%]  (sum=14%)
 artifacts[mma-only]: 7e15001e12c52b3a
 failed assertion sample 1
   balance stat=qps threshold=(≤1.15) ticks=6

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_skewed_cpu_even_ranges_mma.txt

@@ -37,44 +37,44 @@ setting split_queue_enabled=false
 eval duration=35m samples=1 seed=42 cfgs=(sma-count,mma-only,mma-and-count) metrics=(cpu,cpu_util,leases,replicas,write_bytes_per_second)
 ----
 cpu#1: last:  [s1=984642922, s2=1126669982, s3=703798726, s4=419425213, s5=432156047, s6=559114683, s7=142318086, s8=283346689, s9=424209084] (stddev=303225920.55, mean=563964603.56, sum=5075681432)
-cpu#1: thrash_pct: [s1=171%, s2=49%, s3=47%, s4=9%, s5=9%, s6=11%, s7=4%, s8=14%, s9=25%]  (sum=340%)
+cpu#1: thrash_pct: [s1=14%, s2=49%, s3=47%, s4=9%, s5=9%, s6=11%, s7=4%, s8=14%, s9=25%]  (sum=182%)
 cpu_util#1: last:  [s1=0.20, s2=0.23, s3=0.14, s4=0.08, s5=0.09, s6=0.11, s7=0.03, s8=0.06, s9=0.08] (stddev=0.06, mean=0.11, sum=1)
-cpu_util#1: thrash_pct: [s1=171%, s2=49%, s3=47%, s4=9%, s5=9%, s6=11%, s7=4%, s8=14%, s9=25%]  (sum=340%)
+cpu_util#1: thrash_pct: [s1=14%, s2=49%, s3=47%, s4=9%, s5=9%, s6=11%, s7=4%, s8=14%, s9=25%]  (sum=182%)
 leases#1: first: [s1=36, s2=0, s3=0, s4=36, s5=0, s6=0, s7=36, s8=0, s9=0] (stddev=16.97, mean=12.00, sum=108)
 leases#1: last:  [s1=13, s2=12, s3=12, s4=17, s5=11, s6=9, s7=15, s8=10, s9=9] (stddev=2.54, mean=12.00, sum=108)
-leases#1: thrash_pct: [s1=51%, s2=37%, s3=43%, s4=21%, s5=20%, s6=25%, s7=15%, s8=25%, s9=37%]  (sum=274%)
+leases#1: thrash_pct: [s1=53%, s2=38%, s3=44%, s4=21%, s5=20%, s6=26%, s7=15%, s8=26%, s9=37%]  (sum=281%)
 replicas#1: first: [s1=36, s2=36, s3=36, s4=36, s5=36, s6=36, s7=36, s8=36, s9=36] (stddev=0.00, mean=36.00, sum=324)
 replicas#1: last:  [s1=36, s2=37, s3=33, s4=37, s5=36, s6=36, s7=36, s8=35, s9=38] (stddev=1.33, mean=36.00, sum=324)
 replicas#1: thrash_pct: [s1=433%, s2=402%, s3=672%, s4=135%, s5=117%, s6=217%, s7=117%, s8=218%, s9=403%]  (sum=2713%)
 write_bytes_per_second#1: last:  [s1=5619, s2=5639, s3=5275, s4=6269, s5=6137, s6=6150, s7=6187, s8=5861, s9=6290] (stddev=336.10, mean=5936.33, sum=53427)
-write_bytes_per_second#1: thrash_pct: [s1=244%, s2=279%, s3=352%, s4=222%, s5=233%, s6=255%, s7=245%, s8=246%, s9=301%]  (sum=2378%)
+write_bytes_per_second#1: thrash_pct: [s1=572%, s2=656%, s3=832%, s4=513%, s5=540%, s6=593%, s7=564%, s8=563%, s9=701%]  (sum=5535%)
 artifacts[sma-count]: b1d346c990864182
 ==========================
 cpu#1: last:  [s1=572909386, s2=573979147, s3=572126524, s4=557253107, s5=557048324, s6=558469521, s7=571735501, s8=555522023, s9=554214626] (stddev=8138152.97, mean=563695351.00, sum=5073258159)
-cpu#1: thrash_pct: [s1=186%, s2=68%, s3=67%, s4=9%, s5=10%, s6=17%, s7=10%, s8=13%, s9=32%]  (sum=413%)
+cpu#1: thrash_pct: [s1=10%, s2=68%, s3=68%, s4=9%, s5=10%, s6=17%, s7=10%, s8=13%, s9=32%]  (sum=236%)
 cpu_util#1: last:  [s1=0.11, s2=0.11, s3=0.11, s4=0.11, s5=0.11, s6=0.11, s7=0.11, s8=0.11, s9=0.11] (stddev=0.00, mean=0.11, sum=1)
-cpu_util#1: thrash_pct: [s1=186%, s2=68%, s3=67%, s4=9%, s5=10%, s6=17%, s7=10%, s8=13%, s9=32%]  (sum=413%)
+cpu_util#1: thrash_pct: [s1=10%, s2=68%, s3=68%, s4=9%, s5=10%, s6=17%, s7=10%, s8=13%, s9=32%]  (sum=236%)
 leases#1: first: [s1=36, s2=0, s3=0, s4=36, s5=0, s6=0, s7=36, s8=0, s9=0] (stddev=16.97, mean=12.00, sum=108)
 leases#1: last:  [s1=4, s2=4, s3=4, s4=40, s5=4, s6=4, s7=40, s8=4, s9=4] (stddev=14.97, mean=12.00, sum=108)
-leases#1: thrash_pct: [s1=0%, s2=42%, s3=42%, s4=0%, s5=0%, s6=6%, s7=0%, s8=0%, s9=16%]  (sum=106%)
+leases#1: thrash_pct: [s1=0%, s2=42%, s3=42%, s4=0%, s5=0%, s6=6%, s7=0%, s8=0%, s9=17%]  (sum=107%)
 replicas#1: first: [s1=36, s2=36, s3=36, s4=36, s5=36, s6=36, s7=36, s8=36, s9=36] (stddev=0.00, mean=36.00, sum=324)
 replicas#1: last:  [s1=28, s2=28, s3=28, s4=40, s5=40, s6=40, s7=40, s8=40, s9=40] (stddev=5.66, mean=36.00, sum=324)
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=17%, s7=0%, s8=0%, s9=47%]  (sum=64%)
 write_bytes_per_second#1: last:  [s1=3898, s2=3891, s3=3878, s4=6939, s5=6939, s6=6929, s7=6990, s8=6964, s9=6978] (stddev=1446.16, mean=5934.00, sum=53406)
-write_bytes_per_second#1: thrash_pct: [s1=330%, s2=345%, s3=355%, s4=92%, s5=89%, s6=101%, s7=91%, s8=91%, s9=113%]  (sum=1605%)
+write_bytes_per_second#1: thrash_pct: [s1=580%, s2=609%, s3=629%, s4=162%, s5=157%, s6=179%, s7=161%, s8=161%, s9=203%]  (sum=2842%)
 artifacts[mma-only]: dc568a1ea7098ea0
 ==========================
 cpu#1: last:  [s1=572415460, s2=566578888, s3=569096957, s4=556745916, s5=570328363, s6=558184663, s7=560070762, s8=557855218, s9=558141451] (stddev=5893908.39, mean=563268630.89, sum=5069417678)
-cpu#1: thrash_pct: [s1=345%, s2=159%, s3=168%, s4=67%, s5=39%, s6=46%, s7=146%, s8=59%, s9=43%]  (sum=1071%)
+cpu#1: thrash_pct: [s1=180%, s2=159%, s3=168%, s4=66%, s5=39%, s6=45%, s7=145%, s8=58%, s9=43%]  (sum=903%)
 cpu_util#1: last:  [s1=0.11, s2=0.11, s3=0.11, s4=0.11, s5=0.11, s6=0.11, s7=0.11, s8=0.11, s9=0.11] (stddev=0.00, mean=0.11, sum=1)
-cpu_util#1: thrash_pct: [s1=345%, s2=159%, s3=168%, s4=67%, s5=39%, s6=46%, s7=146%, s8=59%, s9=43%]  (sum=1071%)
+cpu_util#1: thrash_pct: [s1=180%, s2=159%, s3=168%, s4=66%, s5=39%, s6=45%, s7=145%, s8=58%, s9=43%]  (sum=903%)
 leases#1: first: [s1=36, s2=0, s3=0, s4=36, s5=0, s6=0, s7=36, s8=0, s9=0] (stddev=16.97, mean=12.00, sum=108)
 leases#1: last:  [s1=7, s2=10, s3=8, s4=17, s5=13, s6=13, s7=16, s8=12, s9=12] (stddev=3.13, mean=12.00, sum=108)
-leases#1: thrash_pct: [s1=145%, s2=127%, s3=121%, s4=68%, s5=32%, s6=32%, s7=136%, s8=55%, s9=43%]  (sum=758%)
+leases#1: thrash_pct: [s1=149%, s2=130%, s3=124%, s4=70%, s5=33%, s6=33%, s7=140%, s8=56%, s9=44%]  (sum=778%)
 replicas#1: first: [s1=36, s2=36, s3=36, s4=36, s5=36, s6=36, s7=36, s8=36, s9=36] (stddev=0.00, mean=36.00, sum=324)
 replicas#1: last:  [s1=36, s2=34, s3=36, s4=37, s5=36, s6=36, s7=37, s8=36, s9=36] (stddev=0.82, mean=36.00, sum=324)
 replicas#1: thrash_pct: [s1=150%, s2=455%, s3=325%, s4=127%, s5=125%, s6=150%, s7=352%, s8=150%, s9=150%]  (sum=1985%)
 write_bytes_per_second#1: last:  [s1=5194, s2=5256, s3=5415, s4=6318, s5=6216, s6=6171, s7=6283, s8=6298, s9=6257] (stddev=461.69, mean=5934.22, sum=53408)
-write_bytes_per_second#1: thrash_pct: [s1=716%, s2=652%, s3=608%, s4=304%, s5=233%, s6=244%, s7=518%, s8=298%, s9=344%]  (sum=3917%)
+write_bytes_per_second#1: thrash_pct: [s1=1721%, s2=1566%, s3=1458%, s4=714%, s5=542%, s6=570%, s7=1227%, s8=694%, s9=803%]  (sum=9293%)
 artifacts[mma-and-count]: 9acfd49d51ee3d3c
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/load_distribution_movement_disabled_enable_later.txt

@@ -45,16 +45,16 @@ setting rebalance_mode=4 delay=2m
 eval duration=7m samples=1 seed=42 cfgs=(sma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
 cpu#1: last:  [s1=451007155, s2=424066775, s3=481923720, s4=487552810, s5=432089126, s6=536127090, s7=582915883, s8=497226435, s9=545078443, s10=559276708] (stddev=51954473.68, mean=499726414.50, sum=4997264145)
-cpu#1: thrash_pct: [s1=131%, s2=148%, s3=144%, s4=128%, s5=132%, s6=15%, s7=13%, s8=6%, s9=12%, s10=14%]  (sum=743%)
+cpu#1: thrash_pct: [s1=19%, s2=33%, s3=40%, s4=24%, s5=15%, s6=16%, s7=14%, s8=7%, s9=12%, s10=14%]  (sum=193%)
 cpu_util#1: last:  [s1=0.03, s2=0.03, s3=0.03, s4=0.03, s5=0.03, s6=0.03, s7=0.04, s8=0.03, s9=0.03, s10=0.03] (stddev=0.00, mean=0.03, sum=0)
-cpu_util#1: thrash_pct: [s1=131%, s2=148%, s3=144%, s4=128%, s5=132%, s6=15%, s7=13%, s8=6%, s9=12%, s10=14%]  (sum=743%)
+cpu_util#1: thrash_pct: [s1=19%, s2=33%, s3=40%, s4=24%, s5=15%, s6=16%, s7=14%, s8=7%, s9=12%, s10=14%]  (sum=193%)
 leases#1: first: [s1=6, s2=6, s3=6, s4=6, s5=6, s6=6, s7=6, s8=6, s9=6, s10=6] (stddev=0.00, mean=6.00, sum=60)
 leases#1: last:  [s1=54, s2=56, s3=55, s4=53, s5=57, s6=58, s7=60, s8=57, s9=60, s10=61] (stddev=2.55, mean=57.10, sum=571)
 leases#1: thrash_pct: [s1=106%, s2=118%, s3=95%, s4=95%, s5=123%, s6=33%, s7=33%, s8=27%, s9=27%, s10=30%]  (sum=688%)
 replicas#1: first: [s1=18, s2=18, s3=18, s4=18, s5=18, s6=18, s7=18, s8=18, s9=18, s10=18] (stddev=0.00, mean=18.00, sum=180)
 replicas#1: last:  [s1=175, s2=172, s3=179, s4=173, s5=173, s6=174, s7=167, s8=169, s9=169, s10=162] (stddev=4.50, mean=171.30, sum=1713)
 replicas#1: thrash_pct: [s1=87%, s2=91%, s3=92%, s4=93%, s5=98%, s6=16%, s7=15%, s8=14%, s9=15%, s10=10%]  (sum=531%)
 write_bytes_per_second#1: last:  [s1=2994621, s2=2991747, s3=3008247, s4=2988902, s5=3006699, s6=2993336, s7=2993993, s8=3002728, s9=2996629, s10=2997216] (stddev=6108.70, mean=2997411.80, sum=29974118)
-write_bytes_per_second#1: thrash_pct: [s1=76%, s2=43%, s3=88%, s4=77%, s5=6%, s6=259%, s7=305%, s8=204%, s9=216%, s10=157%]  (sum=1431%)
+write_bytes_per_second#1: thrash_pct: [s1=73%, s2=40%, s3=85%, s4=74%, s5=5%, s6=160%, s7=207%, s8=103%, s9=114%, s10=55%]  (sum=916%)
 artifacts[sma-count]: ae2919ade8043626
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma_constraint_satisfaction1.txt

@@ -32,9 +32,9 @@ setting split_queue_enabled=false
 eval duration=40m samples=1 seed=42 cfgs=(mma-only) metrics=(cpu,cpu_util,leases,replicas)
 ----
 cpu#1: last:  [s1=36277802, s2=33539839, s3=31138221, s4=499392, s5=499392, s6=499392, s7=0, s8=0, s9=0] (stddev=15793888.65, mean=11383782.00, sum=102454038)
-cpu#1: thrash_pct: [s1=140%, s2=32%, s3=30%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=206%)
+cpu#1: thrash_pct: [s1=8%, s2=32%, s3=30%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=75%)
 cpu_util#1: last:  [s1=725.56, s2=670.80, s3=622.76, s4=9.99, s5=9.99, s6=9.99, s7=0.00, s8=0.00, s9=0.00] (stddev=315.88, mean=227.68, sum=2049)
-cpu_util#1: thrash_pct: [s1=140%, s2=32%, s3=30%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=206%)
+cpu_util#1: thrash_pct: [s1=8%, s2=32%, s3=30%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=75%)
 leases#1: first: [s1=100, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=31.43, mean=11.11, sum=100)
 leases#1: last:  [s1=36, s2=33, s3=31, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=15.76, mean=11.11, sum=100)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma_constraint_satisfaction1_full_disk.txt

@@ -28,12 +28,12 @@ eval duration=60m samples=1 seed=42 cfgs=(mma-only) metrics=(cpu,cpu_util,leases
 ----
 disk_fraction_used#1: first: [s1=0.61, s2=0.61, s3=0.00, s4=0.61, s5=0.61, s6=0.61, s7=0.00, s8=0.00, s9=0.00] (stddev=0.30, mean=0.34, sum=3)
 disk_fraction_used#1: last:  [s1=0.49, s2=0.49, s3=0.24, s4=0.61, s5=0.61, s6=0.61, s7=0.00, s8=0.00, s9=0.00] (stddev=0.26, mean=0.34, sum=3)
-disk_fraction_used#1: thrash_pct: [s1=22%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=22%)
+disk_fraction_used#1: thrash_pct: [s1=25%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=25%)
 leases#1: first: [s1=10, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=3.14, mean=1.11, sum=10)
 leases#1: last:  [s1=7, s2=0, s3=3, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=2.28, mean=1.11, sum=10)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)
 replicas#1: first: [s1=10, s2=10, s3=0, s4=10, s5=10, s6=10, s7=0, s8=0, s9=0] (stddev=4.97, mean=5.56, sum=50)
 replicas#1: last:  [s1=8, s2=8, s3=4, s4=10, s5=10, s6=10, s7=0, s8=0, s9=0] (stddev=4.30, mean=5.56, sum=50)
-replicas#1: thrash_pct: [s1=22%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=22%)
+replicas#1: thrash_pct: [s1=25%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=25%)
 artifacts[mma-only]: 512bcb3481322125
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma_constraint_satisfaction_old_alloc.txt

@@ -29,21 +29,21 @@ set_span_config
 eval duration=60m samples=1 seed=42 metrics=(cpu,cpu_util,leases,replicas) cfgs=(sma-count,mma-only)
 ----
 cpu#1: last:  [s1=41374276, s2=27297814, s3=32347286, s4=499562, s5=499562, s6=499562, s7=0, s8=0, s9=0] (stddev=16111846.58, mean=11390895.78, sum=102518062)
-cpu#1: thrash_pct: [s1=134%, s2=14%, s3=29%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=178%)
+cpu#1: thrash_pct: [s1=14%, s2=14%, s3=29%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=59%)
 cpu_util#1: last:  [s1=827.49, s2=545.96, s3=646.95, s4=9.99, s5=9.99, s6=9.99, s7=0.00, s8=0.00, s9=0.00] (stddev=322.24, mean=227.82, sum=2050)
-cpu_util#1: thrash_pct: [s1=134%, s2=14%, s3=29%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=178%)
+cpu_util#1: thrash_pct: [s1=14%, s2=14%, s3=29%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=59%)
 leases#1: first: [s1=100, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=31.43, mean=11.11, sum=100)
 leases#1: last:  [s1=41, s2=27, s3=32, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=16.07, mean=11.11, sum=100)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=8%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=8%)
 replicas#1: first: [s1=100, s2=100, s3=0, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=49.69, mean=55.56, sum=500)
 replicas#1: last:  [s1=69, s2=67, s3=64, s4=100, s5=100, s6=100, s7=0, s8=0, s9=0] (stddev=41.59, mean=55.56, sum=500)
-replicas#1: thrash_pct: [s1=8%, s2=6%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=13%)
+replicas#1: thrash_pct: [s1=8%, s2=6%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=14%)
 artifacts[sma-count]: b5de9422a7b1ef99
 ==========================
 cpu#1: last:  [s1=36336519, s2=33585339, s3=31110149, s4=500211, s5=500211, s6=500211, s7=0, s8=0, s9=0] (stddev=15807188.65, mean=11392515.56, sum=102532640)
-cpu#1: thrash_pct: [s1=141%, s2=33%, s3=32%, s4=2%, s5=2%, s6=2%, s7=0%, s8=0%, s9=0%]  (sum=211%)
+cpu#1: thrash_pct: [s1=10%, s2=33%, s3=32%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=80%)
 cpu_util#1: last:  [s1=726.73, s2=671.71, s3=622.20, s4=10.00, s5=10.00, s6=10.00, s7=0.00, s8=0.00, s9=0.00] (stddev=316.14, mean=227.85, sum=2051)
-cpu_util#1: thrash_pct: [s1=141%, s2=33%, s3=32%, s4=2%, s5=2%, s6=2%, s7=0%, s8=0%, s9=0%]  (sum=211%)
+cpu_util#1: thrash_pct: [s1=10%, s2=33%, s3=32%, s4=1%, s5=1%, s6=1%, s7=0%, s8=0%, s9=0%]  (sum=80%)
 leases#1: first: [s1=100, s2=0, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=31.43, mean=11.11, sum=100)
 leases#1: last:  [s1=36, s2=33, s3=31, s4=0, s5=0, s6=0, s7=0, s8=0, s9=0] (stddev=15.76, mean=11.11, sum=100)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%]  (sum=0%)