Merge #152506

152506: asim: measure thrashing r=tbg a=tbg

This commit adds a thrashing benchmark to each stat output. "Thrashing" can be
measured in many ways, but here we measure, for each per-store timeseries, a
trend-discounting total variation. Roughly speaking, this computes the upward and
downward total variation separately, then interpolates between the minimum of both
and their sum. When one direction clearly dominates the other, it biases toward the
minimum. When up and down are roughly comparable, it returns something approximating
the sum - which is the "vanilla" total variation. In other words, thrashing is low when the
time series has a clear trend, and it is high when the series moves around but doesn't
seem to be trending anywhere.

When displaying the resulting value, we additionally normalize (divide by) the "range" of values
seen (i.e. the max minus the min), and print the result as a percentage.

For
example, the time series [0, 1, 0, 1, 0] has an upward and downward variation of 2. Neither dominates, so the trend-adjusted total variation equals the total variation 4. The values are all in `[0,1]`, so the range is 1. The resulting trend-discounting total variation percentage would be `tdtv/range = 4/1 = 400%`.

Intuitively, this means that the time series "sweeps" across the range
four times. Additional oscillations would drive this number up further.

In contast, the series `[0, 1, 0, 10, 100, 1000]` is mostly increasing, with the exception of the small downward jump between the second and third datapoints. Consequently, the trend-discounting total variation would (roughly) equal only the downward total variation, which equals 1. The range is 1000, so the percentage in this case would be `1/1000 = 0.1%`.

When printing the thrashing across a set of stores (but for a fixed stat),
instead of computing the range within each store's time series, we compute it
across the entire set. This makes it more likely that we discover the "true
range".

All of this can be made arbitrarily more complicated (and additional
normalization by the number of runs or ticks may be appropriate for some use
cases), but this should be a good start.

Epic: CRDB-49117
Release note: none


Co-authored-by: Tobias Grieger <[email protected]>

Author: craig[bot]

pkg/BUILD.bazel

@@ -240,6 +240,7 @@ ALL_TESTS = [
     "//pkg/kv/kvserver/allocator/storepool:storepool_test",
     "//pkg/kv/kvserver/apply:apply_test",
     "//pkg/kv/kvserver/asim/gossip:gossip_test",
+    "//pkg/kv/kvserver/asim/history:history_test",
     "//pkg/kv/kvserver/asim/metrics:metrics_test",
     "//pkg/kv/kvserver/asim/op:op_test",
     "//pkg/kv/kvserver/asim/queue:queue_test",
@@ -1506,6 +1507,7 @@ GO_TARGETS = [
     "//pkg/kv/kvserver/asim/gossip:gossip",
     "//pkg/kv/kvserver/asim/gossip:gossip_test",
     "//pkg/kv/kvserver/asim/history:history",
+    "//pkg/kv/kvserver/asim/history:history_test",
     "//pkg/kv/kvserver/asim/metrics:metrics",
     "//pkg/kv/kvserver/asim/metrics:metrics_test",
     "//pkg/kv/kvserver/asim/mmaintegration:mmaintegration",

pkg/kv/kvserver/asim/history/BUILD.bazel

@@ -1,8 +1,11 @@
-load("@io_bazel_rules_go//go:def.bzl", "go_library")
+load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test")
 
 go_library(
     name = "history",
-    srcs = ["history.go"],
+    srcs = [
+        "history.go",
+        "thrashing.go",
+    ],
     importpath = "github.com/cockroachdb/cockroach/pkg/kv/kvserver/asim/history",
     visibility = ["//visibility:public"],
     deps = [
@@ -11,3 +14,19 @@ go_library(
         "@com_github_montanaflynn_stats//:stats",
     ],
 )
+
+go_test(
+    name = "history_test",
+    srcs = ["thrashing_test.go"],
+    data = glob(["testdata/**"]),
+    embed = [":history"],
+    deps = [
+        "//pkg/testutils",
+        "//pkg/testutils/datapathutils",
+        "//pkg/testutils/echotest",
+        "//pkg/util/randutil",
+        "@com_github_guptarohit_asciigraph//:asciigraph",
+        "@com_github_stretchr_testify//assert",
+        "@com_github_stretchr_testify//require",
+    ],
+)

pkg/kv/kvserver/asim/history/history.go

@@ -47,6 +47,72 @@ func (h *History) PerStoreValuesAt(idx int, stat string) []float64 {
 	return values
 }
 
+// ThrashingForStat returns a per-store slice of thrashing measurements for the
+// provided stat.
+func (h *History) ThrashingForStat(stat string) ThrashingSlice {
+	if len(h.Recorded) == 0 {
+		return nil
+	}
+	numStores := len(h.PerStoreValuesAt(0, stat))
+	if numStores == 0 {
+		return nil
+	}
+
+	vsByStore := make([][]float64, numStores)
+	for tick := range h.Recorded {
+		for storeIdx, v := range h.PerStoreValuesAt(tick, stat) {
+			vsByStore[storeIdx] = append(vsByStore[storeIdx], v)
+		}
+	}
+
+	ths := make(ThrashingSlice, numStores)
+	for storeIdx := range vsByStore {
+		// HACK: we remove leading zeroes before computeThrasing. This works
+		// around the fact that some timeseries only show sensible values after an
+		// initial period of inactivity. For example, CPU usage is zero until the
+		// first stats tick. Without this hack, the large initial jump from zero to
+		// the first value would be interpreted as variation.
+		th := computeThrashing(stripLeaderingZeroes(vsByStore[storeIdx]))
+		ths[storeIdx] = th
+	}
+	ths.normalize()
+	return ths
+}
+
+func stripLeaderingZeroes(vs []float64) []float64 {
+	for i := range vs {
+		if vs[i] == 0 {
+			continue
+		}
+		return vs[i:]
+	}
+	return nil
+}
+
+// Thrashing returns a string representation of the thrashing for the given
+// stat.
+func (h *History) Thrashing(stat string) string {
+	var buf strings.Builder
+	_, _ = fmt.Fprintf(&buf, "[")
+
+	ths := h.ThrashingForStat(stat)
+	tvpcts := make([]float64, len(ths))
+	for i, th := range ths {
+		if i > 0 {
+			_, _ = fmt.Fprintf(&buf, ", ")
+		}
+		tvpct := th.TDTVPercent()
+		_, _ = fmt.Fprintf(&buf, "s%d=%.0f%%", i+1, tvpct)
+		tvpcts[i] = tvpct
+	}
+	_, _ = fmt.Fprintf(&buf, "] ")
+
+	sum, _ := stats.Sum(tvpcts)
+	_, _ = fmt.Fprintf(&buf, " (sum=%.0f%%)", sum)
+
+	return buf.String()
+}
+
 // ShowRecordedValueAt returns a string representation of the recorded values.
 // The returned boolean is false if (and only if) the recorded values were all
 // zero.

pkg/kv/kvserver/asim/history/testdata/TestTDTV.txt

@@ -0,0 +1,14 @@
+echo
+----
+[0.00 0.26 0.46 0.66 0.83 1.00 0.83 0.66 0.46 0.26 0.00]
+ 0.96 ┤        ╭─╮
+ 0.86 ┤       ╭╯ ╰╮
+ 0.77 ┤      ╭╯   ╰╮
+ 0.67 ┤     ╭╯     ╰╮
+ 0.57 ┤    ╭╯       ╰╮
+ 0.48 ┤   ╭╯         ╰╮
+ 0.38 ┤  ╭╯           ╰╮
+ 0.29 ┤ ╭╯             ╰╮
+ 0.19 ┤ │               │
+ 0.10 ┤╭╯               ╰╮
+ 0.00 ┼╯                 ╰

pkg/kv/kvserver/asim/history/testdata/TestThrashing/bathtub

@@ -0,0 +1,16 @@
+echo
+----
+This sequence has high thrashing, since it ends up where it started.
+input: [10 8 4 2 1 2 4 8 10]
+ 10.00 ┼─╮                                   ╭─
+  9.11 ┤ ╰─╮                               ╭─╯
+  8.22 ┤   ╰─╮                           ╭─╯
+  7.33 ┤     ╰╮                         ╭╯
+  6.44 ┤      ╰╮                       ╭╯
+  5.55 ┤       ╰╮                     ╭╯
+  4.66 ┤        ╰╮                   ╭╯
+  3.77 ┤         ╰─╮               ╭─╯
+  2.88 ┤           ╰──╮         ╭──╯
+  1.99 ┤              ╰──╮   ╭──╯
+  1.10 ┤                 ╰───╯
+tdtv=200.00% (18.0/9.0) uptv=9.0 dntv=9.0 runs=2

pkg/kv/kvserver/asim/history/testdata/TestThrashing/empty

@@ -0,0 +1,4 @@
+echo
+----
+input: []
+tdtv=-0.00% (0.0/-Inf) uptv=0.0 dntv=0.0 runs=1

pkg/kv/kvserver/asim/history/testdata/TestThrashing/final_outlier

@@ -0,0 +1,17 @@
+echo
+----
+An almost monotonic function (the final outlier dominates total variation),
+so it is assigned a small thrashing percentage.
+input: [1 3 2 1 2005]
+ 2005 ┤                                      ╭
+ 1805 ┤                                     ╭╯
+ 1604 ┤                                    ╭╯
+ 1404 ┤                                   ╭╯
+ 1203 ┤                                  ╭╯
+ 1003 ┤                                 ╭╯
+  803 ┤                                ╭╯
+  602 ┤                               ╭╯
+  402 ┤                              ╭╯
+  201 ┤                             ╭╯
+    1 ┼─────────────────────────────╯
+tdtv=0.50% (10.0/2004.0) uptv=2006.0 dntv=2.0 runs=3

pkg/kv/kvserver/asim/history/testdata/TestThrashing/hump

@@ -0,0 +1,16 @@
+echo
+----
+This sequence has high thrashing, since it ends up where it started.
+input: [1 2 4 8 10 8 4 2 1]
+ 9.79 ┤                 ╭───╮
+ 8.92 ┤               ╭─╯   ╰─╮
+ 8.04 ┤             ╭─╯       ╰─╮
+ 7.16 ┤            ╭╯           ╰╮
+ 6.28 ┤           ╭╯             ╰╮
+ 5.40 ┤          ╭╯               ╰╮
+ 4.52 ┤         ╭╯                 ╰╮
+ 3.64 ┤       ╭─╯                   ╰─╮
+ 2.76 ┤     ╭─╯                       ╰─╮
+ 1.88 ┤ ╭───╯                           ╰───╮
+ 1.00 ┼─╯                                   ╰─
+tdtv=200.00% (18.0/9.0) uptv=9.0 dntv=9.0 runs=2

pkg/kv/kvserver/asim/history/testdata/TestThrashing/initial_outlier

@@ -0,0 +1,18 @@
+echo
+----
+An initial outlier leads to a large normalization factor, 
+i.e. low thrashing percentage. This isn't necessarily good, if this becomes
+an issue we could use an inter-quantile range instead.
+input: [10250 13 12 1 2]
+ 10250 ┼╮
+  9225 ┤╰╮
+  8200 ┤ ╰╮
+  7175 ┤  ╰╮
+  6150 ┤   ╰╮
+  5126 ┤    ╰╮
+  4101 ┤     ╰╮
+  3076 ┤      ╰╮
+  2051 ┤       ╰╮
+  1026 ┤        ╰╮
+     1 ┤         ╰─────────────────────────────
+tdtv=0.07% (7.3/10249.0) uptv=1.0 dntv=10249.0 runs=2

pkg/kv/kvserver/asim/history/testdata/TestThrashing/last_point_adds_tv

@@ -0,0 +1,16 @@
+echo
+----
+Regression test to make sure thrashing in only the last index registers.
+input: [2 1 2]
+ 2.00 ┼╮                                     ╭
+ 1.90 ┤╰╮                                   ╭╯
+ 1.81 ┤ ╰─╮                               ╭─╯
+ 1.71 ┤   ╰─╮                           ╭─╯
+ 1.61 ┤     ╰─╮                       ╭─╯
+ 1.51 ┤       ╰─╮                   ╭─╯
+ 1.42 ┤         ╰─╮               ╭─╯
+ 1.32 ┤           ╰─╮           ╭─╯
+ 1.22 ┤             ╰─╮       ╭─╯
+ 1.12 ┤               ╰─╮   ╭─╯
+ 1.03 ┤                 ╰───╯
+tdtv=200.00% (2.0/1.0) uptv=1.0 dntv=1.0 runs=2