asim: improve comments for tests under mma

This test improves the comments on the setup. I’ve made a first pass to clarify
the meaning of various parts, though it may require further review to fully
understand the scenario. This is a starting point.

Author: wenyihu6

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/full_disk.txt

@@ -1,9 +1,21 @@
+# This test verifies that the allocator can rebalance replicas when stores
+# have limited disk capacity and some stores become nearly full. The test
+# sets up a 5-node cluster where each store has a 10GB capacity. It creates
+# 15 ranges with 3 replicas each, where each range is 500MiB in size. The
+# initial placement is skewed, causing stores s1, s2, and s3 to have most
+# replicas (with s1 at 92% capacity, s2 at 85%, and s3 at 49%), while s4
+# and s5 are underutilized at 24% capacity each.
+#
+# Expected outcome: The allocator should rebalance replicas to distribute
+# disk usage evenly across all stores, moving replicas from the fuller
+# stores (s1, s2, s3) to the less utilized stores (s4, s5). The final
+# distribution should achieve balanced disk usage (~55% on each store) and
+# more even replica distribution across all nodes.
 gen_cluster nodes=5 store_byte_capacity_gib=10
 ----
 
-# Each range will be 500 MiB in size and the placement will be skewed, s.t.
-# n1/s1, n2/s2 and n3/s3 will have every replicas initially and n1/s1 will have
-# every lease.
+# Each range will be 500 MiB (524288000 B) in size and the placement will be skewed,
+# s.t. n1/s1, n2/s2 and n3/s3 will have most replicas initially.
 gen_ranges ranges=15 bytes=524288000 repl_factor=3 placement_type=skewed
 ----
 

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/heterogeneous_cpu.txt

@@ -1,3 +1,14 @@
+# This test verifies allocator's behavior in a heterogeneous cluster where nodes
+# have different CPU capacities. The test sets up a 3-node cluster with nodes n1
+# and n2 having 8vcpu capacity each, and node n3 having 16vcpu capacity. It
+# creates 200 ranges evenly distributed across all stores and generates a
+# read-only workload.
+#
+# Expected outcome: mma should balance cpu load based on cpu utilization
+# percentage rather than absolute cpu nanos. However, both mma and sma currently
+# balance on absolute cpu-nanos. n3 should handle more load due to its higher
+# capacity, but the current implementation doesn't account for this. This is
+# tracked in issue: https://github.com/cockroachdb/cockroach/issues/153777.
 gen_cluster nodes=3 node_cpu_rate_capacity=(8000000000,8000000000,16000000000)
 ----
 
@@ -18,7 +29,7 @@ assertion stat=cpu_util type=balance ticks=6 upper_bound=1.1
 ----
 asserting: max_{stores}(cpu_util)/mean_{stores}(cpu_util) ≤ 1.10 at each of last 6 ticks
 
-eval cfgs=(sma-count,mma-only,mma-count) duration=10m metrics=(cpu,cpu_util)
+eval cfgs=(sma-count,mma-count) duration=10m metrics=(cpu,cpu_util)
 ----
 cpu#1: last:  [s1=6700914166, s2=6696073333, s3=6603012499] (stddev=45053658.00, mean=6666666666.00, sum=19999999998)
 cpu#1: thrash_pct: [s1=185%, s2=170%, s3=189%]  (sum=544%)
@@ -38,20 +49,6 @@ cpu#1: last:  [s1=6399850833, s2=6395993333, s3=7203112093] (stddev=379573477.72
 cpu#1: thrash_pct: [s1=32%, s2=37%, s3=40%]  (sum=110%)
 cpu_util#1: last:  [s1=0.80, s2=0.80, s3=0.45] (stddev=0.16, mean=0.68, sum=2)
 cpu_util#1: thrash_pct: [s1=8%, s2=9%, s3=5%]  (sum=21%)
-artifacts[mma-only]: 10d7af8d9edcf883
-failed assertion sample 1
-  balance stat=cpu_util threshold=(≤1.10) ticks=6
-	max/mean=1.17 tick=0
-	max/mean=1.17 tick=1
-	max/mean=1.17 tick=2
-	max/mean=1.17 tick=3
-	max/mean=1.17 tick=4
-	max/mean=1.17 tick=5
-==========================
-cpu#1: last:  [s1=6399850833, s2=6395993333, s3=7203112093] (stddev=379573477.72, mean=6666318753.00, sum=19998956259)
-cpu#1: thrash_pct: [s1=32%, s2=37%, s3=40%]  (sum=110%)
-cpu_util#1: last:  [s1=0.80, s2=0.80, s3=0.45] (stddev=0.16, mean=0.68, sum=2)
-cpu_util#1: thrash_pct: [s1=8%, s2=9%, s3=5%]  (sum=21%)
 artifacts[mma-count]: 10d7af8d9edcf883
 failed assertion sample 1
   balance stat=cpu_util threshold=(≤1.10) ticks=6

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu.txt

@@ -1,6 +1,17 @@
+# This test verifies that the allocator can rebalance replicas and leases when
+# there is high cpu load imbalance across the cluster. The test sets up a 10-node
+# cluster with two distinct workloads: one evenly distributed across all nodes,
+# and another high-cpu workload initially concentrated on only the first few nodes
+# due to skewed placement. The second workload has significantly higher cpu cost
+# per op, creating cpu imbalance.
+#
+# Expected outcome: The allocator should rebalance both replicas and leases to
+# distribute the high-cpu workload more evenly across all 10 nodes.
 gen_cluster nodes=10 node_cpu_rate_capacity=8000000000
 ----
 
+# TODO(wenyihu6): why didn't we balance more replicas/leases - is it because of a very high cpu per range 
+
 # Set the rebalance mode to use the mma store rebalancer and disable the lease
 # and replicate queues so that only the mma store rebalancer is moving replicas
 # or leases.
@@ -15,36 +26,36 @@ gen_load rate=5000 rw_ratio=0.95 min_block=100 max_block=100 request_cpu_per_acc
 ----
 
 # Another workload is added over the second half of the keyspace, which is initially
-# only on s1-s3.
+# mostly on s1-s3.
 gen_ranges ranges=50 min_key=10001 max_key=20000 placement_type=skewed
 ----
 
 gen_load rate=5000 rw_ratio=0.95 min_block=128 max_block=128 request_cpu_per_access=100000 raft_cpu_per_write=20000 min_key=10001 max_key=20000
 ----
 
-eval duration=15m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,write_bytes_per_second,replicas,leases)
+eval duration=2m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,replicas,leases)
 ----
-cpu#1: last:  [s1=274870057, s2=124118783, s3=42166496, s4=21298975, s5=10805903, s6=10577758, s7=453407, s8=10306222, s9=10413474, s10=10503921] (stddev=81956672.84, mean=51551499.60, sum=515514996)
-cpu#1: thrash_pct: [s1=7%, s2=6%, s3=4%, s4=3%, s5=2%, s6=2%, s7=0%, s8=2%, s9=2%, s10=2%]  (sum=30%)
+cpu#1: last:  [s1=275096159, s2=123983362, s3=41814276, s4=21433672, s5=10796253, s6=10602552, s7=439843, s8=10300378, s9=10452776, s10=10595723] (stddev=81999286.66, mean=51551499.40, sum=515514994)
+cpu#1: thrash_pct: [s1=4%, s2=3%, s3=3%, s4=2%, s5=1%, s6=1%, s7=0%, s8=1%, s9=1%, s10=1%]  (sum=18%)
+cpu_util#1: last:  [s1=0.03, s2=0.02, s3=0.01, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00, s10=0.00] (stddev=0.01, mean=0.01, sum=0)
+cpu_util#1: thrash_pct: [s1=4%, s2=3%, s3=3%, s4=2%, s5=1%, s6=1%, s7=0%, s8=1%, s9=1%, s10=1%]  (sum=18%)
 leases#1: first: [s1=37, s2=22, s3=14, s4=13, s5=11, s6=11, s7=10, s8=11, s9=10, s10=11] (stddev=8.07, mean=15.00, sum=150)
 leases#1: last:  [s1=37, s2=22, s3=14, s4=13, s5=11, s6=11, s7=10, s8=11, s9=10, s10=11] (stddev=8.07, mean=15.00, sum=150)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%]  (sum=0%)
 replicas#1: first: [s1=80, s2=70, s3=51, s4=42, s5=37, s6=35, s7=34, s8=33, s9=34, s10=34] (stddev=16.02, mean=45.00, sum=450)
 replicas#1: last:  [s1=80, s2=70, s3=51, s4=42, s5=37, s6=35, s7=34, s8=33, s9=34, s10=34] (stddev=16.02, mean=45.00, sum=450)
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%]  (sum=0%)
-write_bytes_per_second#1: last:  [s1=39511, s2=33080, s3=20899, s4=15208, s5=11942, s6=10699, s7=10093, s8=9465, s9=10055, s10=10043] (stddev=10247.09, mean=17099.50, sum=170995)
-write_bytes_per_second#1: thrash_pct: [s1=13%, s2=18%, s3=20%, s4=19%, s5=17%, s6=16%, s7=14%, s8=14%, s9=17%, s10=16%]  (sum=165%)
-artifacts[mma-only]: bd71a8872f557e0f
+artifacts[mma-only]: c9c14a2b21947e75
 ==========================
-cpu#1: last:  [s1=153545974, s2=82571497, s3=61967377, s4=31436939, s5=21209665, s6=31257441, s7=10903219, s8=40903888, s9=51026201, s10=30714935] (stddev=39256550.41, mean=51553713.60, sum=515537136)
-cpu#1: thrash_pct: [s1=10%, s2=6%, s3=7%, s4=5%, s5=4%, s6=6%, s7=3%, s8=6%, s9=7%, s10=5%]  (sum=58%)
+cpu#1: last:  [s1=153767559, s2=82526536, s3=61655396, s4=31442666, s5=21243662, s6=31483931, s7=10725049, s8=40802943, s9=51247053, s10=30866698] (stddev=39300865.24, mean=51576149.30, sum=515761493)
+cpu#1: thrash_pct: [s1=6%, s2=4%, s3=4%, s4=3%, s5=2%, s6=4%, s7=1%, s8=4%, s9=5%, s10=3%]  (sum=37%)
+cpu_util#1: last:  [s1=0.02, s2=0.01, s3=0.01, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.01, s9=0.01, s10=0.00] (stddev=0.00, mean=0.01, sum=0)
+cpu_util#1: thrash_pct: [s1=6%, s2=4%, s3=4%, s4=3%, s5=2%, s6=4%, s7=1%, s8=4%, s9=5%, s10=3%]  (sum=37%)
 leases#1: first: [s1=37, s2=22, s3=14, s4=13, s5=11, s6=11, s7=10, s8=11, s9=10, s10=11] (stddev=8.07, mean=15.00, sum=150)
 leases#1: last:  [s1=20, s2=16, s3=15, s4=16, s5=12, s6=14, s7=12, s8=15, s9=15, s10=15] (stddev=2.14, mean=15.00, sum=150)
 leases#1: thrash_pct: [s1=0%, s2=0%, s3=15%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%]  (sum=15%)
 replicas#1: first: [s1=80, s2=70, s3=51, s4=42, s5=37, s6=35, s7=34, s8=33, s9=34, s10=34] (stddev=16.02, mean=45.00, sum=450)
 replicas#1: last:  [s1=45, s2=44, s3=44, s4=47, s5=44, s6=44, s7=44, s8=45, s9=46, s10=47] (stddev=1.18, mean=45.00, sum=450)
 replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%, s6=0%, s7=0%, s8=0%, s9=0%, s10=0%]  (sum=0%)
-write_bytes_per_second#1: last:  [s1=25330, s2=20719, s3=18391, s4=17246, s5=15288, s6=15257, s7=14520, s8=14450, s9=15382, s10=14423] (stddev=3361.47, mean=17100.60, sum=171006)
-write_bytes_per_second#1: thrash_pct: [s1=84%, s2=62%, s3=67%, s4=35%, s5=45%, s6=38%, s7=29%, s8=29%, s9=42%, s10=40%]  (sum=471%)
-artifacts[mma-count]: abbd0fc9dbc1971a
+artifacts[mma-count]: de0b265129d19e1
 ==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_25nodes.txt

@@ -1,3 +1,7 @@
+# This test verifies that the allocator can rebalance replicas and leases when
+# there is high CPU load imbalance across a large cluster. The test set-up is
+# similar to high_cpu.txt but is on 25 nodes and with 3x the load for two
+# gen_load commands.
 gen_cluster nodes=25 node_cpu_rate_capacity=8000000000
 ----
 
@@ -12,7 +16,7 @@ gen_load rate=15000 rw_ratio=0.95 min_block=100 max_block=100 request_cpu_per_ac
 ----
 
 # Another workload is added over the second half of the keyspace, which is initially
-# only on s1-s3.
+# only mainly on s1-s3 due to the skewed distribution.
 gen_ranges ranges=50 min_key=10001 max_key=20000 placement_type=skewed
 ----
 

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_able_to_shed_leases.txt

@@ -0,0 +1,56 @@
+# See comments on top of high_cpu_able_to_shed_leases.txt for test details.
+
+# Case (2) where s1 has leases and is CPU overloaded due to raft CPU. It
+# will be able to shed its own leases because it is the leaseholer. There should
+# be a period of lease-rebalancing activity before replica-rebalancing.
+
+gen_cluster nodes=5 node_cpu_rate_capacity=9000000000
+----
+
+setting split_queue_enabled=false
+----
+
+gen_ranges ranges=25 min_key=0 max_key=10000 placement_type=replica_placement
+{s1:*,s2,s3}:7
+{s1:*,s4,s5}:6
+{s1:*,s2,s4}:6
+{s1:*,s3,s5}:6
+----
+{s1:*,s2,s3}:7
+{s1:*,s4,s5}:6
+{s1:*,s2,s4}:6
+{s1:*,s3,s5}:6
+
+gen_load rate=50000 rw_ratio=0 min_key=0 max_key=10000 raft_cpu_per_write=100000
+----
+
+eval duration=30m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
+----
+cpu#1: last:  [s1=2599713435, s2=2799759450, s3=3200050347, s4=3199885256, s5=3200075310] (stddev=253112601.53, mean=2999896759.60, sum=14999483798)
+cpu#1: thrash_pct: [s1=162%, s2=84%, s3=146%, s4=177%, s5=177%]  (sum=746%)
+cpu_util#1: last:  [s1=0.29, s2=0.31, s3=0.36, s4=0.36, s5=0.36] (stddev=0.03, mean=0.33, sum=2)
+cpu_util#1: thrash_pct: [s1=162%, s2=84%, s3=146%, s4=177%, s5=177%]  (sum=746%)
+leases#1: first: [s1=25, s2=0, s3=0, s4=0, s5=0] (stddev=10.00, mean=5.00, sum=25)
+leases#1: last:  [s1=3, s2=5, s3=7, s4=10, s5=0] (stddev=3.41, mean=5.00, sum=25)
+leases#1: thrash_pct: [s1=30%, s2=0%, s3=0%, s4=0%, s5=57%]  (sum=87%)
+replicas#1: first: [s1=25, s2=13, s3=13, s4=12, s5=12] (stddev=5.02, mean=15.00, sum=75)
+replicas#1: last:  [s1=13, s2=14, s3=16, s4=16, s5=16] (stddev=1.26, mean=15.00, sum=75)
+replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=35%]  (sum=35%)
+write_bytes_per_second#1: last:  [s1=25997, s2=27997, s3=32000, s4=31998, s5=32000] (stddev=2530.93, mean=29998.40, sum=149992)
+write_bytes_per_second#1: thrash_pct: [s1=162%, s2=84%, s3=146%, s4=177%, s5=177%]  (sum=746%)
+artifacts[mma-only]: 123731b2fdd740e2
+==========================
+cpu#1: last:  [s1=3200054918, s2=2800070997, s3=3200598149, s4=2799968139, s5=3000353068] (stddev=179022817.20, mean=3000209054.20, sum=15001045271)
+cpu#1: thrash_pct: [s1=125%, s2=63%, s3=50%, s4=108%, s5=80%]  (sum=426%)
+cpu_util#1: last:  [s1=0.36, s2=0.31, s3=0.36, s4=0.31, s5=0.33] (stddev=0.02, mean=0.33, sum=2)
+cpu_util#1: thrash_pct: [s1=125%, s2=63%, s3=50%, s4=108%, s5=80%]  (sum=426%)
+leases#1: first: [s1=25, s2=0, s3=0, s4=0, s5=0] (stddev=10.00, mean=5.00, sum=25)
+leases#1: last:  [s1=6, s2=2, s3=5, s4=5, s5=7] (stddev=1.67, mean=5.00, sum=25)
+leases#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
+replicas#1: first: [s1=25, s2=13, s3=13, s4=12, s5=12] (stddev=5.02, mean=15.00, sum=75)
+replicas#1: last:  [s1=16, s2=14, s3=16, s4=14, s5=15] (stddev=0.89, mean=15.00, sum=75)
+replicas#1: thrash_pct: [s1=0%, s2=0%, s3=0%, s4=0%, s5=0%]  (sum=0%)
+write_bytes_per_second#1: last:  [s1=32000, s2=28000, s3=32005, s4=27999, s5=30003] (stddev=1790.20, mean=30001.40, sum=150007)
+write_bytes_per_second#1: thrash_pct: [s1=124%, s2=63%, s3=50%, s4=108%, s5=80%]  (sum=426%)
+artifacts[mma-count]: 2ef4e5947798976f
+==========================

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_cpu_unable_to_shed_leases.txt

@@ -1,27 +1,27 @@
+# This test verifies that for remotely cpu-overloaded stores, mma wait for lease
+# shedding grace period (remoteStoreLeaseSheddingGraceDuration) before rebalancing
+# replicas away from the store. The test sets up a 5-node cluster where store s1
+# has a high replica count (25 out of 75 total replicas) but holds no leases. All
+# leases are distributed among stores s2-s5. A write-only workload with high raft
+# CPU cost creates CPU pressure primarily on s1 due to its replica count, but s1
+# cannot shed leases since it holds none.
+#
+# Expected outcome:
 # Want to test two cases:
-# (1) Where its impossible to shed leases from the CPU overloaded store, so we
-#     should initially observe a period of no rebalancing activity away from
-#     the store.
-# (2) Where its possible to shed leases from the CPU overloaded store, so we
-#     should observe a period of lease transfers before any replica based
-#     rebalancing away from the store occurs.
+# (1) high_cpu_unable_to_shed_leases.txt: Where its impossible to shed leases
+# from the cpu overloaded s1, so we should initially observe a period of no
+# rebalancing activity away from the store before
+# any replica based rebalancing.
+# (2) high_cpu_able_to_shed_leases.txt: Where its possible to shed leases from
+# the CPU overloaded s1, so we should observe a period of lease transfers before
+# any replica based rebalancing away from the store occurs.
+
 gen_cluster nodes=5 node_cpu_rate_capacity=9000000000
 ----
 
 setting split_queue_enabled=false
 ----
 
-# Case (1) where s1 has no leases and is CPU overloaded due to raft CPU. It
-# won't be able to shed its own replicas because it is not the leaseholder for
-# any of the ranges.
-
-# Originally, this test uses replica_weights=(0.3,0.175,0.175,0.175,0.175)
-# lease_weights=(0,0.25,0.25,0.25,0.25). Replication factor is 3 by default. 75
-# replicas in total. replicas distribution is approximately s1: 23, s2: 13, s3:
-# 13, s4: 13, s5: 13 leaseholder weights: s2: 7 leaseholder, s3: 6 leaseholder,
-# s4: 6 leaseholder, s5: 6 leaseholder. To approximate this, we use replica
-# placement: As an approximation, (s1,s2*,s3):7, (s1,s4,s5*):6, (s1,s2,s4*):6,
-# (s1,s3*,s5):6 s1 does not have the lease. Other stores have the same
 gen_ranges ranges=25 min_key=0 max_key=10000 placement_type=replica_placement
 {s1,s2:*,s3}:7
 {s1,s4,s5:*}:6
@@ -66,5 +66,3 @@ write_bytes_per_second#1: last:  [s1=2600, s2=3202, s3=3201, s4=3000, s5=3000] (
 write_bytes_per_second#1: thrash_pct: [s1=19%, s2=22%, s3=18%, s4=16%, s5=20%]  (sum=96%)
 artifacts[mma-count]: 5b1fca7fda20dfdf
 ==========================
-
-# TODO(kvoli): Case (2)

pkg/kv/kvserver/asim/tests/testdata/non_rand/mma/high_write_uniform_cpu.txt

@@ -1,17 +1,22 @@
+# This test sets up a 10-node cluster with two workloads: (read-only, high-cpu on lh)
+# uniformly across nodes and (write-only, high-write) initially concentrated on s1-s3.
+#
+# Expected outcome: mma should rebalance replicas and leases to distribute the
+# cpu load and write load more evenly across all stores.
 gen_cluster nodes=10 node_cpu_rate_capacity=3000000000
 ----
 
-# Read only workload, which generates 100_000 request cpu nanos/s evenly over
+# Read only workload, which generates 1000 request cpu nanos/s evenly over
 # the first half of the keyspace, which will be on all stores initially.
 gen_ranges ranges=30 min_key=1 max_key=10000 placement_type=even
 ----
 
 gen_load rate=1000 rw_ratio=1.0 request_cpu_per_access=5000000 min_key=1 max_key=10000
 ----
 
-# Write only workload, which generates no CPU and 100_000 (x replication
-# factor) write bytes per second over the second half of the keyspace, which
-# are all on s1-s3 initially.
+# Write only workload, which generates no CPU and 20000op/s*1000B/op =
+# 20000000B/s (x 3 replication factor) write bytes per second over the second half
+# of the keyspace, which are all on s1-s3 initially.
 gen_ranges ranges=30 min_key=10001 max_key=20000 placement_type=skewed
 ----
 
@@ -21,12 +26,15 @@ gen_load rate=20000 rw_ratio=0 min_block=1000 max_block=1000 min_key=10001 max_k
 setting split_queue_enabled=false
 ----
 
-eval duration=20m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(cpu,cpu_util,write_bytes_per_second,replicas,leases)
+eval duration=20m samples=1 seed=42 cfgs=(mma-only,mma-count) metrics=(disk_fraction_used,cpu,cpu_util,write_bytes_per_second,replicas,leases)
 ----
 cpu#1: last:  [s1=501095833, s2=496283333, s3=501825000, s4=499525000, s5=499191666, s6=496991666, s7=497529166, s8=505645833, s9=500662500, s10=501250000] (stddev=2612585.85, mean=499999999.70, sum=4999999997)
 cpu#1: thrash_pct: [s1=462%, s2=503%, s3=518%, s4=521%, s5=478%, s6=550%, s7=472%, s8=510%, s9=542%, s10=507%]  (sum=5065%)
 cpu_util#1: last:  [s1=0.17, s2=0.17, s3=0.17, s4=0.17, s5=0.17, s6=0.17, s7=0.17, s8=0.17, s9=0.17, s10=0.17] (stddev=0.00, mean=0.17, sum=2)
 cpu_util#1: thrash_pct: [s1=462%, s2=503%, s3=518%, s4=521%, s5=478%, s6=550%, s7=472%, s8=510%, s9=542%, s10=507%]  (sum=5065%)
+disk_fraction_used#1: first: [s1=0.00, s2=0.00, s3=0.00, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00, s10=0.00] (stddev=0.00, mean=0.00, sum=0)
+disk_fraction_used#1: last:  [s1=0.03, s2=0.03, s3=0.03, s4=0.03, s5=0.03, s6=0.03, s7=0.03, s8=0.03, s9=0.03, s10=0.03] (stddev=0.00, mean=0.03, sum=0)
+disk_fraction_used#1: thrash_pct: [s1=119%, s2=55%, s3=43%, s4=45%, s5=27%, s6=23%, s7=24%, s8=12%, s9=109%, s10=29%]  (sum=487%)
 leases#1: first: [s1=19, s2=11, s3=6, s4=3, s5=3, s6=3, s7=4, s8=3, s9=4, s10=4] (stddev=4.92, mean=6.00, sum=60)
 leases#1: last:  [s1=10, s2=6, s3=6, s4=3, s5=6, s6=4, s7=4, s8=6, s9=9, s10=6] (stddev=2.05, mean=6.00, sum=60)
 leases#1: thrash_pct: [s1=16%, s2=15%, s3=38%, s4=12%, s5=14%, s6=0%, s7=25%, s8=14%, s9=42%, s10=27%]  (sum=204%)
@@ -41,6 +49,9 @@ cpu#1: last:  [s1=666962932, s2=499499610, s3=667678881, s4=502782759, s5=330971
 cpu#1: thrash_pct: [s1=309%, s2=335%, s3=369%, s4=290%, s5=323%, s6=198%, s7=502%, s8=252%, s9=78%, s10=247%]  (sum=2904%)
 cpu_util#1: last:  [s1=0.22, s2=0.17, s3=0.22, s4=0.17, s5=0.11, s6=0.17, s7=0.17, s8=0.06, s9=0.17, s10=0.22] (stddev=0.05, mean=0.17, sum=2)
 cpu_util#1: thrash_pct: [s1=309%, s2=335%, s3=369%, s4=290%, s5=323%, s6=198%, s7=502%, s8=252%, s9=78%, s10=247%]  (sum=2904%)
+disk_fraction_used#1: first: [s1=0.00, s2=0.00, s3=0.00, s4=0.00, s5=0.00, s6=0.00, s7=0.00, s8=0.00, s9=0.00, s10=0.00] (stddev=0.00, mean=0.00, sum=0)
+disk_fraction_used#1: last:  [s1=0.04, s2=0.03, s3=0.04, s4=0.03, s5=0.03, s6=0.03, s7=0.04, s8=0.03, s9=0.03, s10=0.03] (stddev=0.00, mean=0.03, sum=0)
+disk_fraction_used#1: thrash_pct: [s1=224%, s2=272%, s3=323%, s4=271%, s5=184%, s6=250%, s7=394%, s8=278%, s9=110%, s10=164%]  (sum=2470%)
 leases#1: first: [s1=19, s2=11, s3=6, s4=3, s5=3, s6=3, s7=4, s8=3, s9=4, s10=4] (stddev=4.92, mean=6.00, sum=60)
 leases#1: last:  [s1=8, s2=8, s3=8, s4=7, s5=5, s6=4, s7=7, s8=1, s9=5, s10=7] (stddev=2.14, mean=6.00, sum=60)
 leases#1: thrash_pct: [s1=154%, s2=214%, s3=224%, s4=171%, s5=135%, s6=157%, s7=270%, s8=124%, s9=101%, s10=159%]  (sum=1709%)