microbench-ci: compiler induced variance threshold

Previously, thresholds were removed in favor of better p-value tuning. However,
this did not account for reproducible compiler induced variance. In these
situations the binaries reliably reproduce a regression even though the code
change is unrelated. To account for this, we now check if the delta is less than
a threshold we consider the metric to have no change.

The thresholds were determined heuristically by running the benchmarks with
and without the change and looking at the results.

Author: herkolategan

pkg/cmd/microbench-ci/benchmark.go

@@ -28,12 +28,17 @@ type (
 		Iterations   int      `yaml:"iterations"`
 		CompareAlpha float64  `yaml:"compare_alpha"`
 		Retries      int      `yaml:"retries"`
-		Metrics      []string `yaml:"metrics"`
+		Metrics      []Metric `yaml:"metrics"`
 	}
 	Benchmarks  []Benchmark
 	ProfileType string
 )
 
+type Metric struct {
+	Name      string  `yaml:"name"`
+	Threshold float64 `yaml:"threshold"`
+}
+
 const (
 	ProfileCPU    ProfileType = "cpu"
 	ProfileMemory ProfileType = "memory"

pkg/cmd/microbench-ci/compare.go

@@ -9,6 +9,7 @@ import (
 	"bufio"
 	"bytes"
 	"fmt"
+	"math"
 	"os"
 	"path"
 
@@ -66,6 +67,22 @@ func (c *CompareResult) status(metricName string) Status {
 	} else if cc.Delta*float64(entry.Better) < 0 {
 		status = Regressed
 	}
+
+	// Check if the metric has a delta cutoff threshold.
+	threshold := 0.0
+	for _, metric := range c.Benchmark.Metrics {
+		if metric.Name == metricName {
+			threshold = metric.Threshold
+			break
+		}
+	}
+	// If the threshold is set and the delta is less than the threshold, we
+	// consider the metric to have no change. This accounts for compiler induced
+	// variance, where the regression might be reproducible, but the change is
+	// unrelated to the changes in the code.
+	if math.Abs(cc.Delta) < threshold {
+		status = NoChange
+	}
 	return status
 }
 

pkg/cmd/microbench-ci/config/pull-request-suite.yml

@@ -9,8 +9,10 @@ benchmarks:
     compare_alpha: 0.025
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: .3
+      - name: "allocs/op"
+        threshold: .1
 
   - display_name: Sysbench
     labels: ["KV", "3node", "oltp_read_only"]
@@ -22,8 +24,10 @@ benchmarks:
     compare_alpha: 0.025
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: .4
+      - name: "allocs/op"
+        threshold: .1
 
   - display_name: Sysbench
     labels: ["KV", "3node", "oltp_write_only"]
@@ -35,5 +39,7 @@ benchmarks:
     compare_alpha: 0.025
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: .4
+      - name: "allocs/op"
+        threshold: .1

pkg/cmd/microbench-ci/report.go

@@ -59,7 +59,8 @@ func (c *CompareResult) generateSummaryData(
 	statusTemplateFunc func(status Status) string,
 ) []SummaryData {
 	summaryData := make([]SummaryData, 0, len(c.MetricMap))
-	for _, metricName := range c.Benchmark.Metrics {
+	for _, metric := range c.Benchmark.Metrics {
+		metricName := metric.Name
 		entry := c.MetricMap[metricName]
 		if entry == nil {
 			log.Printf("WARN: no metric found for benchmark metric %q", metricName)

pkg/cmd/microbench-ci/testdata/regression.txt

@@ -11,8 +11,10 @@ benchmarks:
     compare_alpha: 0.05
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: 0.005
+      - name: "allocs/op"
+        threshold: 0.002
 
 ----
 

pkg/cmd/microbench-ci/testdata/summary.txt

@@ -11,8 +11,10 @@ benchmarks:
     compare_alpha: 0.05
     retries: 3
     metrics:
-      - "sec/op"
-      - "allocs/op"
+      - name: "sec/op"
+        threshold: 0.001
+      - name: "allocs/op"
+        threshold: 0.001
 
 ----