asim: record cpu utilization in StoreMetrics

Previously, StoreMetrics didn't include CPU utilization, making it hard to
evaluate the simulation setup. This commit adds CPU utilization to the store
metrics. Note that due to the lack of node-level metrics, the same CPU capacity
will be repeated for stores on the same node.

Author: wenyihu6

pkg/kv/kvserver/asim/history/history.go

@@ -65,7 +65,7 @@ func (h *History) ShowRecordedValueAt(idx int, stat string) (string, bool) {
 		}
 		storeID := h.Recorded[idx][i].StoreID
 
-		if stat == "disk_fraction_used" {
+		if stat == "disk_fraction_used" || stat == "cpu_util" {
 			_, _ = fmt.Fprintf(&buf, "s%v=%.2f", storeID, v)
 		} else {
 			_, _ = fmt.Fprintf(&buf, "s%v=%.0f", storeID, v)

pkg/kv/kvserver/asim/metrics/series.go

@@ -22,6 +22,7 @@ func MakeTS(metrics [][]StoreMetrics) map[string][][]float64 {
 	// are partially duplicated with the cluster tracker.
 	ret["qps"] = make([][]float64, stores)
 	ret["cpu"] = make([][]float64, stores)
+	ret["cpu_util"] = make([][]float64, stores)
 	ret["write"] = make([][]float64, stores)
 	ret["write_b"] = make([][]float64, stores)
 	ret["write_bytes_per_second"] = make([][]float64, stores)
@@ -40,6 +41,7 @@ func MakeTS(metrics [][]StoreMetrics) map[string][][]float64 {
 		for i, sm := range sms {
 			ret["qps"][i] = append(ret["qps"][i], float64(sm.QPS))
 			ret["cpu"][i] = append(ret["cpu"][i], float64(sm.CPU))
+			ret["cpu_util"][i] = append(ret["cpu_util"][i], sm.NodeCPUUtilization)
 			ret["write"][i] = append(ret["write"][i], float64(sm.WriteKeys))
 			ret["write_b"][i] = append(ret["write_b"][i], float64(sm.WriteBytes))
 			ret["write_bytes_per_second"][i] = append(ret["write_bytes_per_second"][i], float64(sm.WriteBytesPerSecond))

pkg/kv/kvserver/asim/metrics/tracker.go

@@ -22,6 +22,7 @@ type StoreMetrics struct {
 	StoreID             int64
 	QPS                 int64
 	CPU                 int64
+	NodeCPUUtilization  float64
 	WriteKeys           int64
 	WriteBytes          int64
 	WriteBytesPerSecond int64
@@ -75,6 +76,8 @@ func (sm *StoreMetrics) GetMetricValue(stat string) float64 {
 		return float64(sm.RangeSplits)
 	case "disk_fraction_used":
 		return sm.DiskFractionUsed
+	case "cpu_util":
+		return sm.NodeCPUUtilization
 	default:
 		return 0
 	}
@@ -139,12 +142,21 @@ func (mt *Tracker) Tick(ctx context.Context, tick time.Time, s state.State) {
 		}
 
 		desc := store.Descriptor()
+		nodeCapacity := s.NodeCapacity(store.NodeID())
+
+		// NodeCPURateUsage is the same as StoresCPURate in asim.
+		if nodeCapacity.NodeCPURateCapacity == 0 {
+			panic(fmt.Sprintf("unexpected: node cpu rate capacity is 0 (node cpu rate usage = %d)",
+				nodeCapacity.NodeCPURateUsage))
+		}
+		cpuUtil := float64(nodeCapacity.NodeCPURateUsage) / float64(nodeCapacity.NodeCPURateCapacity)
 
 		sm := StoreMetrics{
 			Tick:                tick,
 			StoreID:             int64(storeID),
 			QPS:                 int64(desc.Capacity.QueriesPerSecond),
 			CPU:                 int64(desc.Capacity.CPUPerSecond),
+			NodeCPUUtilization:  cpuUtil,
 			WriteKeys:           u.WriteKeys,
 			WriteBytes:          u.WriteBytes,
 			WriteBytesPerSecond: int64(desc.Capacity.WriteBytesPerSecond),

pkg/kv/kvserver/asim/tests/testdata/generated/example_rebalancing/example_rebalancing_mma-only_1.json

@@ -28,7 +28,7 @@ assertion type=conformance under=0 over=0 unavailable=0 violating=0
 
 eval duration=20m cfgs=(sma-count,mma-only)
 ----
-artifacts[sma-count]: 1375509107ab02af
+artifacts[sma-count]: 19128202c9e746ab
 ==========================
-artifacts[mma-only]: 398e8cd95b3327fb
+artifacts[mma-only]: d80019be362b72f7
 ==========================

pkg/kv/kvserver/asim/tests/testdata/generated/example_rebalancing/example_rebalancing_mma-only_2.json

@@ -34,7 +34,7 @@ eval duration=20m samples=1 seed=42 metrics=(replicas)
 ----
 replicas#1: first: [s1=301, s2=0, s3=0] (stddev=141.89, mean=100.33, sum=301)
 replicas#1: last:  [s1=301, s2=271, s3=267] (stddev=15.17, mean=279.67, sum=839)
-artifacts[sma-count]: 866426beb5ad043b
+artifacts[sma-count]: 2ec16a0a9e1fe5b1
 failed assertion sample 1
   balance stat=replicas threshold=(≤1.05) ticks=6
 	max/mean=1.08 tick=0

pkg/kv/kvserver/asim/tests/testdata/generated/example_rebalancing/example_rebalancing_sma-count_1.json

@@ -28,7 +28,7 @@ assertion type=conformance under=0 over=0 unavailable=0 violating=0
 
 eval duration=10m cfgs=(sma-count,mma-only)
 ----
-artifacts[sma-count]: ca2c6e7399a9dc9c
+artifacts[sma-count]: 1dc992474ba09e56
 ==========================
-artifacts[mma-only]: 273dc0cd62dfba6c
+artifacts[mma-only]: 412f90f16a6ace6
 ==========================