Merge #143262

143262: microbench-ci: benchmark suite tuning r=DarrylWong a=herkolategan

Previously, one node local KV microbenchmarks were run as part of CI. These
tests are sensitive to compiler induced variance. Even though the regressions /
improvements are reproducible, often the code change is unrelated to the
performance change. It happens often enough with these benchmarks that it could
be wasteful for engineers to investigate these slight differences.

This change updates the benchmark suite to run the 3 node KV benchmarks instead
which is less likely to be triggered due to compiler induced variance.

Epic: None
Release note: None

Co-authored-by: Herko Lategan <[email protected]>

Author: craig[bot]

pkg/cmd/microbench-ci/benchmark.go

@@ -28,12 +28,17 @@ type (
 		Iterations   int      `yaml:"iterations"`
 		CompareAlpha float64  `yaml:"compare_alpha"`
 		Retries      int      `yaml:"retries"`
-		Metrics      []string `yaml:"metrics"`
+		Metrics      []Metric `yaml:"metrics"`
 	}
 	Benchmarks  []Benchmark
 	ProfileType string
 )
 
+type Metric struct {
+	Name      string  `yaml:"name"`
+	Threshold float64 `yaml:"threshold"`
+}
+
 const (
 	ProfileCPU    ProfileType = "cpu"
 	ProfileMemory ProfileType = "memory"

pkg/cmd/microbench-ci/compare.go

@@ -9,6 +9,7 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
+	"math"
 	"os"
 	"path"
 
@@ -66,6 +67,22 @@ func (c *CompareResult) status(metricName string) Status {
 	} else if cc.Delta*float64(entry.Better) < 0 {
 		status = Regressed
 	}
+
+	// Check if the metric has a delta cutoff threshold.
+	threshold := 0.0
+	for _, metric := range c.Benchmark.Metrics {
+		if metric.Name == metricName {
+			threshold = metric.Threshold
+			break
+		}
+	}
+	// If the threshold is set and the delta is less than the threshold, we
+	// consider the metric to have no change. This accounts for compiler induced
+	// variance, where the regression might be reproducible, but the change is
+	// unrelated to the changes in the code.
+	if math.Abs(cc.Delta) < threshold {
+		status = NoChange
+	}
 	return status
 }
 

pkg/cmd/microbench-ci/config/pull-request-suite.yml

@@ -9,31 +9,37 @@ benchmarks:
     compare_alpha: 0.025
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: .3
+      - name: "allocs/op"
+        threshold: .1
 
   - display_name: Sysbench
-    labels: ["KV", "1node", "local", "oltp_read_only"]
-    name: "BenchmarkSysbench/KV/1node_local/oltp_read_only"
+    labels: ["KV", "3node", "oltp_read_only"]
+    name: "BenchmarkSysbench/KV/3node/oltp_read_only"
     package: "pkg/sql/tests"
     runner_group: 2
-    count: 20
-    iterations: 6000
+    count: 15
+    iterations: 3000
     compare_alpha: 0.025
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: .4
+      - name: "allocs/op"
+        threshold: .1
 
   - display_name: Sysbench
-    labels: ["KV", "1node", "local", "oltp_write_only"]
-    name: "BenchmarkSysbench/KV/1node_local/oltp_write_only"
+    labels: ["KV", "3node", "oltp_write_only"]
+    name: "BenchmarkSysbench/KV/3node/oltp_write_only"
     package: "pkg/sql/tests"
     runner_group: 2
-    count: 20
-    iterations: 6000
+    count: 15
+    iterations: 3000
     compare_alpha: 0.025
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: .4
+      - name: "allocs/op"
+        threshold: .1

pkg/cmd/microbench-ci/report.go

@@ -59,7 +59,8 @@ func (c *CompareResult) generateSummaryData(
 	statusTemplateFunc func(status Status) string,
 ) []SummaryData {
 	summaryData := make([]SummaryData, 0, len(c.MetricMap))
-	for _, metricName := range c.Benchmark.Metrics {
+	for _, metric := range c.Benchmark.Metrics {
+		metricName := metric.Name
 		entry := c.MetricMap[metricName]
 		if entry == nil {
 			log.Printf("WARN: no metric found for benchmark metric %q", metricName)

pkg/cmd/microbench-ci/testdata/regression.txt

@@ -11,8 +11,10 @@ benchmarks:
     compare_alpha: 0.05
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: 0.005
+      - name: "allocs/op"
+        threshold: 0.002
 
 ----
 

pkg/cmd/microbench-ci/testdata/summary.txt

@@ -11,8 +11,10 @@ benchmarks:
     compare_alpha: 0.05
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: 0.001
+      - name: "allocs/op"
+        threshold: 0.001
 
 ----