asim: change tdtv to support scaling property

When reviewing testdata changes, I realized that sometimes the trend-adjusted
total variation could increase even though the up and down variation had
independently become smaller. This behavior is uninutitive and undesirable, and
I traced it back to the "boosting" of the interpolation parameter I had
introduced in an earlier commit. This situation is now rectified.

Author: tbg

pkg/kv/kvserver/asim/history/testdata/TestTDTV.txt

@@ -1,14 +1,14 @@
 echo
 ----
-[0.00 0.11 0.25 0.43 0.67 1.00 0.67 0.43 0.25 0.11 0.00]
- 0.91 ┤        ╭─╮
- 0.82 ┤        │ │
- 0.73 ┤       ╭╯ ╰╮
- 0.64 ┤       │   │
- 0.55 ┤      ╭╯   ╰╮
- 0.46 ┤     ╭╯     ╰╮
- 0.36 ┤    ╭╯       ╰╮
- 0.27 ┤   ╭╯         ╰╮
- 0.18 ┤  ╭╯           ╰╮
- 0.09 ┤╭─╯             ╰─╮
+[0.00 0.26 0.46 0.66 0.83 1.00 0.83 0.66 0.46 0.26 0.00]
+ 0.96 ┤        ╭─╮
+ 0.86 ┤       ╭╯ ╰╮
+ 0.77 ┤      ╭╯   ╰╮
+ 0.67 ┤     ╭╯     ╰╮
+ 0.57 ┤    ╭╯       ╰╮
+ 0.48 ┤   ╭╯         ╰╮
+ 0.38 ┤  ╭╯           ╰╮
+ 0.29 ┤ ╭╯             ╰╮
+ 0.19 ┤ │               │
+ 0.10 ┤╭╯               ╰╮
  0.00 ┼╯                 ╰

pkg/kv/kvserver/asim/history/thrashing.go

@@ -90,14 +90,18 @@ func computeThrashing(values []float64) thrashing {
 // - tdtv(u,0) = tdtv(0,u) = 0
 // - tdtv(u,u) = 2t
 // - tdtv(ku,kd) = k*tdtv(u,d) for k>=0
+// - tdtv(l*ku, kd) > tdtv(ku) for l>1 (same for kd)
 func tdtv(tu, td float64) float64 {
 	tmin := min(tu, td)
 	if tmin == 0 {
 		// There's only one direction of movement, so we discount all variation.
 		return 0
 	}
 	frac := tmin / max(tu, td) // in [0, 1]
-	alpha := frac * frac       // exponent 2 makes it more "trend-sticky"
+	// The exponent can't exceed 1 because that would violate the scaling property
+	// (last property above). For the endpoint 1, tdtv=2min(tu,td). The choice of
+	// 0.8 is somewhat arbitrary, but gives a reasonable trade-off.
+	alpha := math.Pow(frac, 0.8)
 	return alpha*(tu+td) + (1-alpha)*tmin
 }
 

pkg/kv/kvserver/asim/history/trashing_test.go

@@ -148,6 +148,7 @@ func TestTDTV(t *testing.T) {
 			assert.InDelta(t, orig, scaled, delta)
 			assert.InDelta(t, 0, tdtv(u, 0), delta)
 			assert.InDelta(t, 2*u, tdtv(u, u), delta)
+			assert.Greater(t, tdtv(2*u, d), tdtv(u, d))
 
 			return true
 		}, &quick.Config{MaxCount: 1000, Rand: r}), "seed: %d", seed)

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_fulldisk.txt

@@ -24,17 +24,17 @@ eval duration=30m seed=42 metrics=(replicas,disk_fraction_used) cfgs=(sma-count,
 ----
 disk_fraction_used#1: first: [s1=0.20, s2=0.20, s3=0.20, s4=0.20, s5=1.05] (stddev=0.34, mean=0.37, sum=2)
 disk_fraction_used#1: last:  [s1=0.30, s2=0.30, s3=0.30, s4=0.30, s5=0.95] (stddev=0.26, mean=0.43, sum=2)
-disk_fraction_used#1: thrash_pct: [s1=25%, s2=20%, s3=24%, s4=24%, s5=98%]  (sum=190%)
+disk_fraction_used#1: thrash_pct: [s1=33%, s2=28%, s3=32%, s4=32%, s5=110%]  (sum=235%)
 replicas#1: first: [s1=300, s2=300, s3=300, s4=300, s5=300] (stddev=0.00, mean=300.00, sum=1500)
 replicas#1: last:  [s1=322, s2=328, s3=327, s4=323, s5=200] (stddev=50.05, mean=300.00, sum=1500)
-replicas#1: thrash_pct: [s1=243%, s2=199%, s3=231%, s4=237%, s5=41%]  (sum=951%)
+replicas#1: thrash_pct: [s1=262%, s2=221%, s3=253%, s4=256%, s5=72%]  (sum=1063%)
 artifacts[sma-count]: f36a73f72b67ff15
 ==========================
 disk_fraction_used#1: first: [s1=0.20, s2=0.20, s3=0.20, s4=0.20, s5=1.05] (stddev=0.34, mean=0.37, sum=2)
 disk_fraction_used#1: last:  [s1=0.31, s2=0.31, s3=0.31, s4=0.29, s5=0.92] (stddev=0.25, mean=0.43, sum=2)
-disk_fraction_used#1: thrash_pct: [s1=1%, s2=0%, s3=0%, s4=0%, s5=53%]  (sum=54%)
+disk_fraction_used#1: thrash_pct: [s1=2%, s2=1%, s3=1%, s4=0%, s5=65%]  (sum=69%)
 replicas#1: first: [s1=300, s2=300, s3=300, s4=300, s5=300] (stddev=0.00, mean=300.00, sum=1500)
 replicas#1: last:  [s1=331, s2=329, s3=329, s4=316, s5=195] (stddev=52.77, mean=300.00, sum=1500)
-replicas#1: thrash_pct: [s1=8%, s2=3%, s3=2%, s4=0%, s5=0%]  (sum=14%)
+replicas#1: thrash_pct: [s1=16%, s2=7%, s3=6%, s4=0%, s5=0%]  (sum=29%)
 artifacts[mma-only]: 4bbec1a9787ddb71
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_io_overload.txt

@@ -18,10 +18,10 @@ eval duration=10m seed=42 metrics=(replicas,leases) cfgs=(sma-count,mma-only)
 ----
 leases#1: first: [s1=259, s2=129, s3=64, s4=32, s5=16] (stddev=88.43, mean=100.00, sum=500)
 leases#1: last:  [s1=180, s2=102, s3=106, s4=106, s5=6] (stddev=55.34, mean=100.00, sum=500)
-leases#1: thrash_pct: [s1=12%, s2=16%, s3=12%, s4=15%, s5=0%]  (sum=56%)
+leases#1: thrash_pct: [s1=23%, s2=23%, s3=20%, s4=27%, s5=0%]  (sum=93%)
 replicas#1: first: [s1=500, s2=456, s3=262, s4=165, s5=117] (stddev=153.29, mean=300.00, sum=1500)
 replicas#1: last:  [s1=347, s2=343, s3=348, s4=345, s5=117] (stddev=91.52, mean=300.00, sum=1500)
-replicas#1: thrash_pct: [s1=28%, s2=28%, s3=31%, s4=28%, s5=0%]  (sum=115%)
+replicas#1: thrash_pct: [s1=47%, s2=44%, s3=46%, s4=48%, s5=0%]  (sum=184%)
 artifacts[sma-count]: 937f126429994d65
 failed assertion sample 1
   stat=replicas value=(=0.00) ticks=5

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_liveness.txt

@@ -36,17 +36,17 @@ eval duration=12m seed=42 metrics=(replicas,leases) cfgs=(sma-count,mma-only)
 ----
 leases#1: first: [s1=100, s2=100, s3=100, s4=100, s5=100, s6=100, s7=100] (stddev=0.00, mean=100.00, sum=700)
 leases#1: last:  [s1=134, s2=142, s3=143, s4=139, s5=142, s6=0, s7=0] (stddev=63.31, mean=100.00, sum=700)
-leases#1: thrash_pct: [s1=4%, s2=0%, s3=4%, s4=1%, s5=2%, s6=10%, s7=0%]  (sum=21%)
+leases#1: thrash_pct: [s1=9%, s2=0%, s3=9%, s4=2%, s5=6%, s6=23%, s7=0%]  (sum=48%)
 replicas#1: first: [s1=300, s2=300, s3=300, s4=300, s5=300, s6=300, s7=300] (stddev=0.00, mean=300.00, sum=2100)
 replicas#1: last:  [s1=432, s2=415, s3=420, s4=418, s5=415, s6=0, s7=0] (stddev=189.81, mean=300.00, sum=2100)
-replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=6%, s7=0%]  (sum=6%)
+replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=16%, s7=0%]  (sum=16%)
 artifacts[sma-count]: 6ea963e52d5dbdcd
 ==========================
 leases#1: first: [s1=100, s2=100, s3=100, s4=100, s5=100, s6=100, s7=100] (stddev=0.00, mean=100.00, sum=700)
 leases#1: last:  [s1=134, s2=139, s3=140, s4=151, s5=136, s6=0, s7=0] (stddev=63.44, mean=100.00, sum=700)
-leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=10%, s7=0%]  (sum=10%)
+leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=22%, s7=0%]  (sum=22%)
 replicas#1: first: [s1=300, s2=300, s3=300, s4=300, s5=300, s6=300, s7=300] (stddev=0.00, mean=300.00, sum=2100)
 replicas#1: last:  [s1=431, s2=415, s3=418, s4=419, s5=417, s6=0, s7=0] (stddev=189.80, mean=300.00, sum=2100)
-replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=5%, s7=0%]  (sum=5%)
+replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=14%, s7=0%]  (sum=14%)
 artifacts[mma-only]: e00a4da1eb04fbff
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_multi_store.txt

@@ -23,12 +23,12 @@ eval duration=5m seed=42 metrics=(leases) cfgs=(sma-count,mma-only)
 ----
 leases#1: first: [s1=8, s2=3, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=1, s10=0, s11=1, s12=0, s13=1, s14=0] (stddev=2.10, mean=1.00, sum=14)
 leases#1: last:  [s1=1, s2=1, s3=1, s4=1, s5=1, s6=1, s7=1, s8=1, s9=1, s10=1, s11=1, s12=1, s13=1, s14=1] (stddev=0.00, mean=1.00, sum=14)
-leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=19%, s9=25%, s10=19%, s11=25%, s12=0%, s13=50%, s14=19%]  (sum=156%)
+leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=27%, s9=25%, s10=27%, s11=25%, s12=0%, s13=50%, s14=27%]  (sum=181%)
 artifacts[sma-count]: 6322c40ff6c7b47c
 ==========================
 leases#1: first: [s1=8, s2=3, s3=0, s4=0, s5=0, s6=0, s7=0, s8=0, s9=1, s10=0, s11=1, s12=0, s13=1, s14=0] (stddev=2.10, mean=1.00, sum=14)
 leases#1: last:  [s1=4, s2=1, s3=0, s4=0, s5=0, s6=1, s7=0, s8=0, s9=1, s10=1, s11=1, s12=1, s13=2, s14=2] (stddev=1.07, mean=1.00, sum=14)
-leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=25%, s9=0%, s10=0%, s11=25%, s12=19%, s13=0%, s14=0%]  (sum=69%)
+leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=25%, s9=0%, s10=0%, s11=25%, s12=27%, s13=0%, s14=0%]  (sum=77%)
 artifacts[mma-only]: ed1b3ca87bbb8148
 failed assertion sample 1
   balance stat=leases threshold=(≤1.00) ticks=6

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing.txt

@@ -42,19 +42,19 @@ asserting: |qps(t)/mean_{T}(qps) - 1| ≤ 0.05 ∀ t∈T and each store (T=last
 eval duration=3m samples=2 seed=42 metrics=(qps,replica_moves) cfgs=(sma-count,mma-only)
 ----
 qps#1: last:  [s1=998, s2=998, s3=1001, s4=995, s5=998, s6=1002, s7=1001] (stddev=2.27, mean=999.00, sum=6993)
-qps#1: thrash_pct: [s1=116%, s2=55%, s3=2%, s4=54%, s5=2%, s6=2%, s7=1%]  (sum=233%)
+qps#1: thrash_pct: [s1=144%, s2=77%, s3=6%, s4=76%, s5=6%, s6=5%, s7=4%]  (sum=319%)
 replica_moves#1: last:  [s1=2, s2=1, s3=0, s4=1, s5=1, s6=1, s7=0] (stddev=0.64, mean=0.86, sum=6)
 replica_moves#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%]  (sum=0%)
 qps#2: last:  [s1=996, s2=996, s3=1000, s4=998, s5=1001, s6=1003, s7=1000] (stddev=2.42, mean=999.14, sum=6994)
-qps#2: thrash_pct: [s1=115%, s2=53%, s3=2%, s4=2%, s5=2%, s6=54%, s7=2%]  (sum=230%)
+qps#2: thrash_pct: [s1=143%, s2=75%, s3=5%, s4=5%, s5=5%, s6=76%, s7=5%]  (sum=314%)
 replica_moves#2: last:  [s1=2, s2=1, s3=0, s4=0, s5=0, s6=1, s7=0] (stddev=0.73, mean=0.57, sum=4)
 replica_moves#2: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%]  (sum=0%)
 artifacts[sma-count]: 26755534702067c5
 ==========================
 qps#1: last:  [s1=3999, s2=1997, s3=0, s4=0, s5=0, s6=1003, s7=0] (stddev=1413.61, mean=999.86, sum=6999)
-qps#1: thrash_pct: [s1=2%, s2=2%, s3=0%, s4=0%, s5=0%, s6=1%, s7=0%]  (sum=6%)
+qps#1: thrash_pct: [s1=8%, s2=5%, s3=0%, s4=0%, s5=0%, s6=4%, s7=0%]  (sum=17%)
 qps#2: last:  [s1=3996, s2=1996, s3=0, s4=0, s5=0, s6=1006, s7=0] (stddev=1412.60, mean=999.71, sum=6998)
-qps#2: thrash_pct: [s1=2%, s2=2%, s3=0%, s4=0%, s5=0%, s6=2%, s7=0%]  (sum=5%)
+qps#2: thrash_pct: [s1=7%, s2=4%, s3=0%, s4=0%, s5=0%, s6=5%, s7=0%]  (sum=16%)
 artifacts[mma-only]: dad89302cc72743e
 failed assertion sample 1
   balance stat=qps threshold=(≤1.15) ticks=6
@@ -90,19 +90,19 @@ setting gossip_delay=20s
 eval duration=5m samples=2 seed=42 metrics=(qps,replica_moves) cfgs=(sma-count,mma-only)
 ----
 qps#1: last:  [s1=994, s2=1004, s3=1002, s4=1002, s5=1000, s6=997, s7=999] (stddev=3.15, mean=999.71, sum=6998)
-qps#1: thrash_pct: [s1=233%, s2=88%, s3=2%, s4=42%, s5=88%, s6=88%, s7=40%]  (sum=582%)
+qps#1: thrash_pct: [s1=259%, s2=109%, s3=5%, s4=59%, s5=109%, s6=109%, s7=57%]  (sum=707%)
 replica_moves#1: last:  [s1=6, s2=4, s3=0, s4=1, s5=3, s6=5, s7=2] (stddev=2.00, mean=3.00, sum=21)
 replica_moves#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%]  (sum=0%)
 qps#2: last:  [s1=999, s2=998, s3=998, s4=998, s5=1000, s6=999, s7=1003] (stddev=1.67, mean=999.29, sum=6995)
-qps#2: thrash_pct: [s1=483%, s2=234%, s3=41%, s4=42%, s5=335%, s6=237%, s7=336%]  (sum=1708%)
+qps#2: thrash_pct: [s1=510%, s2=259%, s3=58%, s4=59%, s5=361%, s6=262%, s7=363%]  (sum=1873%)
 replica_moves#2: last:  [s1=15, s2=13, s3=5, s4=3, s5=10, s6=9, s7=8] (stddev=3.89, mean=9.00, sum=63)
 replica_moves#2: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%]  (sum=0%)
 artifacts[sma-count]: d6940fa02dce5639
 ==========================
 qps#1: last:  [s1=3995, s2=1998, s3=0, s4=0, s5=0, s6=1005, s7=0] (stddev=1412.50, mean=999.71, sum=6998)
-qps#1: thrash_pct: [s1=3%, s2=2%, s3=0%, s4=0%, s5=0%, s6=2%, s7=0%]  (sum=7%)
+qps#1: thrash_pct: [s1=8%, s2=6%, s3=0%, s4=0%, s5=0%, s6=4%, s7=0%]  (sum=19%)
 qps#2: last:  [s1=4000, s2=1995, s3=0, s4=0, s5=0, s6=1004, s7=0] (stddev=1413.71, mean=999.86, sum=6999)
-qps#2: thrash_pct: [s1=2%, s2=2%, s3=0%, s4=0%, s5=0%, s6=2%, s7=0%]  (sum=6%)
+qps#2: thrash_pct: [s1=7%, s2=5%, s3=0%, s4=0%, s5=0%, s6=5%, s7=0%]  (sum=17%)
 artifacts[mma-only]: 7e15001e12c52b3a
 failed assertion sample 1
   balance stat=qps threshold=(≤1.15) ticks=6

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_skewed_cpu_even_ranges_mma.txt

@@ -37,44 +37,44 @@ setting split_queue_enabled=false
 eval duration=35m samples=1 seed=42 cfgs=(sma-count,mma-only,mma-and-count) metrics=(cpu,cpu_util,leases,replicas,write_bytes_per_second)
 ----
 cpu#1: last:  [s1=984642922, s2=1126669982, s3=703798726, s4=419425213, s5=432156047, s6=559114683, s7=142318086, s8=283346689, s9=424209084] (stddev=303225920.55, mean=563964603.56, sum=5075681432)
-cpu#1: thrash_pct: [s1=150%, s2=34%, s3=35%, s4=5%, s5=5%, s6=6%, s7=3%, s8=10%, s9=18%]  (sum=267%)
+cpu#1: thrash_pct: [s1=171%, s2=49%, s3=47%, s4=9%, s5=9%, s6=11%, s7=4%, s8=14%, s9=25%]  (sum=340%)
 cpu_util#1: last:  [s1=0.20, s2=0.23, s3=0.14, s4=0.08, s5=0.09, s6=0.11, s7=0.03, s8=0.06, s9=0.08] (stddev=0.06, mean=0.11, sum=1)
-cpu_util#1: thrash_pct: [s1=150%, s2=34%, s3=35%, s4=5%, s5=5%, s6=6%, s7=3%, s8=10%, s9=18%]  (sum=267%)
+cpu_util#1: thrash_pct: [s1=171%, s2=49%, s3=47%, s4=9%, s5=9%, s6=11%, s7=4%, s8=14%, s9=25%]  (sum=340%)
 leases#1: first: [s1=36, s2=0, s3=0, s4=36, s5=0, s6=0, s7=36, s8=0, s9=0] (stddev=16.97, mean=12.00, sum=108)
 leases#1: last:  [s1=13, s2=12, s3=12, s4=17, s5=11, s6=9, s7=15, s8=10, s9=9] (stddev=2.54, mean=12.00, sum=108)
-leases#1: thrash_pct: [s1=28%, s2=22%, s3=27%, s4=9%, s5=10%, s6=15%, s7=6%, s8=14%, s9=23%]  (sum=155%)
+leases#1: thrash_pct: [s1=51%, s2=37%, s3=43%, s4=21%, s5=20%, s6=25%, s7=15%, s8=25%, s9=37%]  (sum=274%)
 replicas#1: first: [s1=36, s2=36, s3=36, s4=36, s5=36, s6=36, s7=36, s8=36, s9=36] (stddev=0.00, mean=36.00, sum=324)
 replicas#1: last:  [s1=36, s2=37, s3=33, s4=37, s5=36, s6=36, s7=36, s8=35, s9=38] (stddev=1.33, mean=36.00, sum=324)
-replicas#1: thrash_pct: [s1=433%, s2=392%, s3=643%, s4=126%, s5=117%, s6=217%, s7=117%, s8=209%, s9=385%]  (sum=2638%)
+replicas#1: thrash_pct: [s1=433%, s2=402%, s3=672%, s4=135%, s5=117%, s6=217%, s7=117%, s8=218%, s9=403%]  (sum=2713%)
 write_bytes_per_second#1: last:  [s1=5619, s2=5639, s3=5275, s4=6269, s5=6137, s6=6150, s7=6187, s8=5861, s9=6290] (stddev=336.10, mean=5936.33, sum=53427)
-write_bytes_per_second#1: thrash_pct: [s1=185%, s2=216%, s3=288%, s4=160%, s5=171%, s6=191%, s7=181%, s8=185%, s9=232%]  (sum=1808%)
+write_bytes_per_second#1: thrash_pct: [s1=244%, s2=279%, s3=352%, s4=222%, s5=233%, s6=255%, s7=245%, s8=246%, s9=301%]  (sum=2378%)
 artifacts[sma-count]: b1d346c990864182
 ==========================
 cpu#1: last:  [s1=572909386, s2=573979147, s3=572126524, s4=557253107, s5=557048324, s6=558469521, s7=571735501, s8=555522023, s9=554214626] (stddev=8138152.97, mean=563695351.00, sum=5073258159)
-cpu#1: thrash_pct: [s1=174%, s2=58%, s3=57%, s4=5%, s5=6%, s6=11%, s7=6%, s8=8%, s9=24%]  (sum=348%)
+cpu#1: thrash_pct: [s1=186%, s2=68%, s3=67%, s4=9%, s5=10%, s6=17%, s7=10%, s8=13%, s9=32%]  (sum=413%)
 cpu_util#1: last:  [s1=0.11, s2=0.11, s3=0.11, s4=0.11, s5=0.11, s6=0.11, s7=0.11, s8=0.11, s9=0.11] (stddev=0.00, mean=0.11, sum=1)
-cpu_util#1: thrash_pct: [s1=174%, s2=58%, s3=57%, s4=5%, s5=6%, s6=11%, s7=6%, s8=8%, s9=24%]  (sum=348%)
+cpu_util#1: thrash_pct: [s1=186%, s2=68%, s3=67%, s4=9%, s5=10%, s6=17%, s7=10%, s8=13%, s9=32%]  (sum=413%)
 leases#1: first: [s1=36, s2=0, s3=0, s4=36, s5=0, s6=0, s7=36, s8=0, s9=0] (stddev=16.97, mean=12.00, sum=108)
 leases#1: last:  [s1=4, s2=4, s3=4, s4=40, s5=4, s6=4, s7=40, s8=4, s9=4] (stddev=14.97, mean=12.00, sum=108)
-leases#1: thrash_pct: [s1=0%, s2=33%, s3=33%, s4=0%, s5=0%, s6=3%, s7=0%, s8=0%, s9=11%]  (sum=80%)
+leases#1: thrash_pct: [s1=0%, s2=42%, s3=42%, s4=0%, s5=0%, s6=6%, s7=0%, s8=0%, s9=16%]  (sum=106%)
 replicas#1: first: [s1=36, s2=36, s3=36, s4=36, s5=36, s6=36, s7=36, s8=36, s9=36] (stddev=0.00, mean=36.00, sum=324)
 replicas#1: last:  [s1=28, s2=28, s3=28, s4=40, s5=40, s6=40, s7=40, s8=40, s9=40] (stddev=5.66, mean=36.00, sum=324)
-replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=9%, s7=0%, s8=0%, s9=31%]  (sum=39%)
+replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=17%, s7=0%, s8=0%, s9=47%]  (sum=64%)
 write_bytes_per_second#1: last:  [s1=3898, s2=3891, s3=3878, s4=6939, s5=6939, s6=6929, s7=6990, s8=6964, s9=6978] (stddev=1446.16, mean=5934.00, sum=53406)
-write_bytes_per_second#1: thrash_pct: [s1=280%, s2=294%, s3=304%, s4=52%, s5=51%, s6=59%, s7=51%, s8=52%, s9=68%]  (sum=1211%)
+write_bytes_per_second#1: thrash_pct: [s1=330%, s2=345%, s3=355%, s4=92%, s5=89%, s6=101%, s7=91%, s8=91%, s9=113%]  (sum=1605%)
 artifacts[mma-only]: dc568a1ea7098ea0
 ==========================
 cpu#1: last:  [s1=572415460, s2=566578888, s3=569096957, s4=556745916, s5=570328363, s6=558184663, s7=560070762, s8=557855218, s9=558141451] (stddev=5893908.39, mean=563268630.89, sum=5069417678)
-cpu#1: thrash_pct: [s1=331%, s2=146%, s3=155%, s4=56%, s5=30%, s6=36%, s7=133%, s8=49%, s9=34%]  (sum=968%)
+cpu#1: thrash_pct: [s1=345%, s2=159%, s3=168%, s4=67%, s5=39%, s6=46%, s7=146%, s8=59%, s9=43%]  (sum=1071%)
 cpu_util#1: last:  [s1=0.11, s2=0.11, s3=0.11, s4=0.11, s5=0.11, s6=0.11, s7=0.11, s8=0.11, s9=0.11] (stddev=0.00, mean=0.11, sum=1)
-cpu_util#1: thrash_pct: [s1=331%, s2=146%, s3=155%, s4=56%, s5=30%, s6=36%, s7=133%, s8=49%, s9=34%]  (sum=968%)
+cpu_util#1: thrash_pct: [s1=345%, s2=159%, s3=168%, s4=67%, s5=39%, s6=46%, s7=146%, s8=59%, s9=43%]  (sum=1071%)
 leases#1: first: [s1=36, s2=0, s3=0, s4=36, s5=0, s6=0, s7=36, s8=0, s9=0] (stddev=16.97, mean=12.00, sum=108)
 leases#1: last:  [s1=7, s2=10, s3=8, s4=17, s5=13, s6=13, s7=16, s8=12, s9=12] (stddev=3.13, mean=12.00, sum=108)
-leases#1: thrash_pct: [s1=97%, s2=103%, s3=101%, s4=42%, s5=18%, s6=18%, s7=98%, s8=36%, s9=27%]  (sum=538%)
+leases#1: thrash_pct: [s1=145%, s2=127%, s3=121%, s4=68%, s5=32%, s6=32%, s7=136%, s8=55%, s9=43%]  (sum=758%)
 replicas#1: first: [s1=36, s2=36, s3=36, s4=36, s5=36, s6=36, s7=36, s8=36, s9=36] (stddev=0.00, mean=36.00, sum=324)
 replicas#1: last:  [s1=36, s2=34, s3=36, s4=37, s5=36, s6=36, s7=37, s8=36, s9=36] (stddev=0.82, mean=36.00, sum=324)
-replicas#1: thrash_pct: [s1=150%, s2=428%, s3=325%, s4=115%, s5=125%, s6=150%, s7=338%, s8=150%, s9=150%]  (sum=1930%)
+replicas#1: thrash_pct: [s1=150%, s2=455%, s3=325%, s4=127%, s5=125%, s6=150%, s7=352%, s8=150%, s9=150%]  (sum=1985%)
 write_bytes_per_second#1: last:  [s1=5194, s2=5256, s3=5415, s4=6318, s5=6216, s6=6171, s7=6283, s8=6298, s9=6257] (stddev=461.69, mean=5934.22, sum=53408)
-write_bytes_per_second#1: thrash_pct: [s1=640%, s2=577%, s3=532%, s4=233%, s5=170%, s6=180%, s7=435%, s8=228%, s9=271%]  (sum=3266%)
+write_bytes_per_second#1: thrash_pct: [s1=716%, s2=652%, s3=608%, s4=304%, s5=233%, s6=244%, s7=518%, s8=298%, s9=344%]  (sum=3917%)
 artifacts[mma-and-count]: 9acfd49d51ee3d3c
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_zone_config.txt

@@ -29,7 +29,7 @@ eval duration=10m samples=1 seed=42 metrics=(replicas) cfgs=(sma-count,mma-only)
 ----
 replicas#1: first: [s1=17, s2=17, s3=17, s4=17, s5=17, s6=16, s7=16, s8=17, s9=17, s10=17, s11=17, s12=17, s13=16, s14=16, s15=16, s16=16, s17=17, s18=17, s19=17, s20=16, s21=17, s22=16, s23=17, s24=17, s25=17, s26=17, s27=17, s28=16, s29=17, s30=16, s31=17, s32=17, s33=16, s34=16, s35=17, s36=17] (stddev=0.47, mean=16.67, sum=600)
 replicas#1: last:  [s1=51, s2=50, s3=52, s4=51, s5=49, s6=49, s7=50, s8=49, s9=49, s10=50, s11=48, s12=52, s13=0, s14=0, s15=0, s16=0, s17=0, s18=0, s19=0, s20=0, s21=0, s22=0, s23=0, s24=0, s25=0, s26=0, s27=0, s28=0, s29=0, s30=0, s31=0, s32=0, s33=0, s34=0, s35=0, s36=0] (stddev=23.58, mean=16.67, sum=600)
-replicas#1: thrash_pct: [s1=13%, s2=0%, s3=6%, s4=0%, s5=4%, s6=13%, s7=0%, s8=13%, s9=6%, s10=9%, s11=13%, s12=21%, s13=0%, s14=0%, s15=0%, s16=0%, s17=0%, s18=0%, s19=0%, s20=0%, s21=0%, s22=0%, s23=0%, s24=0%, s25=0%, s26=0%, s27=0%, s28=0%, s29=0%, s30=0%, s31=0%, s32=0%, s33=0%, s34=0%, s35=0%, s36=0%]  (sum=99%)
+replicas#1: thrash_pct: [s1=28%, s2=0%, s3=15%, s4=0%, s5=11%, s6=28%, s7=0%, s8=28%, s9=15%, s10=20%, s11=28%, s12=41%, s13=0%, s14=0%, s15=0%, s16=0%, s17=0%, s18=0%, s19=0%, s20=0%, s21=0%, s22=0%, s23=0%, s24=0%, s25=0%, s26=0%, s27=0%, s28=0%, s29=0%, s30=0%, s31=0%, s32=0%, s33=0%, s34=0%, s35=0%, s36=0%]  (sum=215%)
 artifacts[sma-count]: d14df5e462a2e42b
 ==========================
 replicas#1: first: [s1=17, s2=17, s3=17, s4=17, s5=17, s6=16, s7=16, s8=17, s9=17, s10=17, s11=17, s12=17, s13=16, s14=16, s15=16, s16=16, s17=17, s18=17, s19=17, s20=16, s21=17, s22=16, s23=17, s24=17, s25=17, s26=17, s27=17, s28=16, s29=17, s30=16, s31=17, s32=17, s33=16, s34=16, s35=17, s36=17] (stddev=0.47, mean=16.67, sum=600)