asim: add explanatory output to assert command

I find that I have to check my understanding on this regularly,
so perhaps this output can help understand that `balance` checks
at each point in time that stores are close together, whereas `steady`
checks that for each store, its min/max does not deviate from its mean (over time),
in other words that the metric only changes within a small band. (Which does not
imply any kind of convergence across the cluster; each store might settle on its
own value for the metric).

Author: tbg

pkg/kv/kvserver/asim/tests/datadriven_simulation_test.go

@@ -580,23 +580,35 @@ func TestDataDriven(t *testing.T) {
 				var ticks int
 				scanMustExist(t, d, "type", &typ)
 
+				var buf strings.Builder
 				switch typ {
 				case "balance":
 					scanMustExist(t, d, "stat", &stat)
 					scanMustExist(t, d, "ticks", &ticks)
+					threshold := scanThreshold(t, d)
 					assertions = append(assertions, assertion.BalanceAssertion{
 						Ticks:     ticks,
 						Stat:      stat,
-						Threshold: scanThreshold(t, d),
+						Threshold: threshold,
 					})
+					_, _ = fmt.Fprintf(&buf, "asserting: max_{stores}(%s)/mean_{stores}(%s) %s %.2f at each of last %d ticks",
+						stat, stat, threshold.ThresholdType, threshold.Value, ticks)
+					// ^-- the max and mean are taken over the stores (with the tick fixed).
 				case "steady":
 					scanMustExist(t, d, "stat", &stat)
 					scanMustExist(t, d, "ticks", &ticks)
+					threshold := scanThreshold(t, d)
 					assertions = append(assertions, assertion.SteadyStateAssertion{
 						Ticks:     ticks,
 						Stat:      stat,
-						Threshold: scanThreshold(t, d),
+						Threshold: threshold,
 					})
+					_, _ = fmt.Fprintf(&buf, "asserting: |%s(t)/mean_{T}(%s) - 1| %s %.2f ∀ t∈T and each store ("+
+						"T=last %d ticks)",
+						stat, stat, threshold.ThresholdType, threshold.Value, ticks)
+					// ^-- the mean is taken over the ticks (and the check runs for each store).
+					// These assertions are for "checking that change stops" (vs. balance
+					// assertions, which verify that stores are close together on some metric).
 				case "stat":
 					var stores []int
 					scanMustExist(t, d, "stat", &stat)
@@ -633,7 +645,7 @@ func TestDataDriven(t *testing.T) {
 				default:
 					panic("unknown assertion: " + typ)
 				}
-				return ""
+				return buf.String()
 			case "setting":
 				// NB: delay could be supported for the below settings,
 				// but it hasn't been needed yet.

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_add_node.txt

@@ -19,6 +19,7 @@ add_node
 # Assert that the replica counts balance within 5% of each other among stores.
 assertion type=balance stat=replicas ticks=6 upper_bound=1.05
 ----
+asserting: max_{stores}(replicas)/mean_{stores}(replicas) < 1.05 at each of last 6 ticks
 
 # Update the replication factor for the keyspace to be 3, instead of the
 # initial replication factor of 1 set during generation.

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_multi_store.txt

@@ -13,9 +13,11 @@ gen_load rate=7000 min_block=512 max_block=512
 
 assertion stat=leases type=balance ticks=6 upper_bound=1
 ----
+asserting: max_{stores}(leases)/mean_{stores}(leases) < 1.00 at each of last 6 ticks
 
 assertion stat=leases type=steady ticks=6 upper_bound=0
 ----
+asserting: |leases(t)/mean_{T}(leases) - 1| < 0.00 ∀ t∈T and each store (T=last 6 ticks)
 
 eval duration=5m seed=42 metrics=(leases) cfgs=(sma-count,mma-only)
 ----

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_rebalancing.txt

@@ -21,6 +21,7 @@ gen_load rate=7000 rw_ratio=0.95 access_skew=false min_block=128 max_block=256
 # seconds) the max/mean QPS of the cluster does not exceed 1.15.
 assertion stat=qps type=balance ticks=6 upper_bound=1.15
 ----
+asserting: max_{stores}(qps)/mean_{stores}(qps) < 1.15 at each of last 6 ticks
 
 # The second is a steady state assertion. The steady state assertion requires
 # that during the last 6 ticks (60 seconds), the value of QPS per-store doesn't
@@ -31,6 +32,7 @@ assertion stat=qps type=balance ticks=6 upper_bound=1.15
 # to take a duration, not ticks.
 assertion stat=qps type=steady ticks=6 upper_bound=0.05
 ----
+asserting: |qps(t)/mean_{T}(qps) - 1| < 0.05 ∀ t∈T and each store (T=last 6 ticks)
 
 # The generators are then called and 2 simulation runs, named samples are
 # created and evaluated. Each sample has a fixed duration of 3 minutes.

pkg/kv/kvserver/asim/tests/testdata/non_rand/example_splitting.txt

@@ -22,6 +22,7 @@ setting split_qps_threshold=2500
 # ticks of the simulation.
 assertion stat=replicas type=steady ticks=6 upper_bound=0.00
 ----
+asserting: |replicas(t)/mean_{T}(replicas) - 1| < 0.00 ∀ t∈T and each store (T=last 6 ticks)
 
 
 # Examine the number of replicas. Here there were 5 load based splits. This