Merge #142933 #142935

142933: roachtest: create go bench output in sysbench r=tbg a=tbg

This makes it easier to compare sysbench results using the standard benchmark
comparison facilities afforded to us by the excellent `benchstat`[^1].

Previously, we'd resort to bash snippets to manufacture a Go benchmark line.
Now, they're simply a part of the test artifacts and (after having run against
both SHAs as many times as desired) you can run

```
find ./artifacts/<beforesha> -name bench.txt | xargs cat > before.txt
find ./artifacts/<aftersha> -name bench.txt | xargs cat > after.txt

benchstat before.txt after.txt
```

Here's an example `bench.txt`:

```
BenchmarkSysbenchSettings/a=oltp_read_write/nodes=3/cpu=8/conc=64	1	27732.02 queries/sec	1386.60 txns/sec	11.92 ms/min	46.15 ms/avg	70.55 ms/p95	157.01 ms/max
```

[^1]: https://pkg.go.dev/golang.org/x/perf/cmd/benchstat

Epic: CRDB-42584
Release note: None


142935: tests: also collect run-only mutex profile in sysbench r=tbg a=tbg

This extends `Benchmark{,Parallel}Sysbench`'s functionality to create a delta
profile for allocations to the mutex profiler.

Epic: CRDB-42584
Release note: none


Co-authored-by: Tobias Grieger <[email protected]>

Author: craig[bot]

pkg/cmd/roachtest/tests/BUILD.bazel

@@ -342,6 +342,8 @@ go_library(
         "@org_golang_google_protobuf//proto",
         "@org_golang_x_exp//maps",
         "@org_golang_x_oauth2//clientcredentials",
+        "@org_golang_x_text//cases",
+        "@org_golang_x_text//language",
     ],
 )
 
@@ -353,6 +355,7 @@ go_test(
         "drt_test.go",
         "query_comparison_util_test.go",
         "restore_test.go",
+        "sysbench_test.go",
         "tpcc_test.go",
         ":mocks_drt",  # keep
     ],

pkg/cmd/roachtest/tests/sysbench.go

@@ -12,6 +12,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"os"
+	"path/filepath"
 	"regexp"
 	"strings"
 	"time"
@@ -29,6 +30,8 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/errors"
 	"github.com/stretchr/testify/require"
+	"golang.org/x/text/cases"
+	"golang.org/x/text/language"
 )
 
 type sysbenchWorkload int
@@ -239,7 +242,22 @@ func runSysbench(ctx context.Context, t test.Test, c cluster.Cluster, opts sysbe
 			return errors.Errorf("no SQL statistics found in sysbench output:\n%s", result.Stdout)
 		}
 		t.L().Printf("sysbench results:\n%s", result.Stdout[idx:])
-		return exportSysbenchResults(t, c, result.Stdout, start, opts)
+
+		if err := exportSysbenchResults(t, c, result.Stdout, start, opts); err != nil {
+			return err
+		}
+
+		// Also produce standard Go benchmark output. This can be used to run
+		// benchstat comparisons.
+		goBenchOutput, err := sysbenchToGoBench(t.Name(), result.Stdout[idx:])
+		if err != nil {
+			return err
+		}
+		if err := os.WriteFile(filepath.Join(t.ArtifactsDir(), "bench.txt"), []byte(goBenchOutput), 0666); err != nil {
+			return err
+		}
+
+		return nil
 	}
 	if opts.usePostgres {
 		if err := runWorkload(ctx); err != nil {
@@ -515,3 +533,78 @@ func detectSysbenchCrash(result install.RunResultDetails) (string, bool) {
 	}
 	return "", false
 }
+
+// sysbenchToGoBench converts sysbench output into Go benchmark format.
+func sysbenchToGoBench(name string, result string) (string, error) {
+	// Extract key metrics from sysbench output using regex patterns.
+	var qps, tps string
+	var minLat, avgLat, p95Lat, maxLat string
+
+	// Parse transactions per second.
+	m := regexp.MustCompile(`transactions:\s+\d+\s+\(([\d.]+)\s+per sec`).FindStringSubmatch(result)
+	if len(m) <= 1 {
+		return "", errors.New("failed to parse transactions per second")
+	}
+	tps = m[1]
+
+	// Parse queries per second.
+	m = regexp.MustCompile(`queries:\s+\d+\s+\(([\d.]+)\s+per sec`).FindStringSubmatch(result)
+	if len(m) <= 1 {
+		return "", errors.New("failed to parse queries per second")
+	}
+	qps = m[1]
+
+	// Parse each latency metric using a loop.
+	metrics := map[string]*string{
+		"min":             &minLat,
+		"avg":             &avgLat,
+		"max":             &maxLat,
+		"95th percentile": &p95Lat,
+	}
+	for metric, ptr := range metrics {
+		pattern := fmt.Sprintf(`%s:\s+([\d.]+)`, metric)
+		m = regexp.MustCompile(pattern).FindStringSubmatch(result)
+		if len(m) <= 1 {
+			return "", errors.Newf("failed to parse %s latency", metric)
+		}
+		*ptr = m[1]
+	}
+
+	// Process the test name.
+	parts := strings.Split(name, "/")
+	if len(parts) == 0 {
+		return "", errors.New("empty test name")
+	}
+
+	// Normalize first segment (e.g. "sysbench-settings" -> "SysbenchSettings").
+	firstPart := parts[0]
+	// Split on non-alphanumeric characters.
+	words := regexp.MustCompile(`[^a-zA-Z0-9]+`).Split(firstPart, -1)
+	// Capitalize each word and join them.
+	var sb strings.Builder
+	for _, word := range words {
+		if word == "" {
+			continue
+		}
+		sb.WriteString(cases.Title(language.Und).String(strings.ToLower(word)))
+	}
+	firstPart = sb.String()
+
+	// Build the benchmark name.
+	benchName := "Benchmark" + firstPart
+
+	// Add remaining parts, using auto-assigned keys only for parts without keys.
+	nextKey := 'a'
+	for _, part := range parts[1:] {
+		if strings.Contains(part, "=") {
+			benchName += "/" + part
+		} else {
+			benchName += fmt.Sprintf("/%s=%s", string(nextKey), part)
+			nextKey++
+		}
+	}
+
+	// Return formatted benchmark string with all metrics.
+	return fmt.Sprintf("%s\t1\t%s queries/sec\t%s txns/sec\t%s ms/min\t%s ms/avg\t%s ms/p95\t%s ms/max",
+		benchName, qps, tps, minLat, avgLat, p95Lat, maxLat), nil
+}

pkg/cmd/roachtest/tests/sysbench_test.go

@@ -0,0 +1,74 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package tests
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+func TestSysbenchToGoBench(t *testing.T) {
+	testCases := []struct {
+		name           string
+		result         string
+		benchmarkName  string
+		expectedOutput string
+		expectError    bool
+	}{
+		{
+			name: "valid",
+			result: `SQL statistics:
+    queries performed:
+        read:                            10711078
+        write:                           3060308
+        other:                           1530154
+        total:                           15301540
+    transactions:                        765077 (1275.05 per sec.)
+    queries:                             15301540 (25501.03 per sec.)
+    ignored errors:                      0      (0.00 per sec.)
+    reconnects:                          0      (0.00 per sec.)
+
+General statistics:
+    total time:                          600.0347s
+    total number of events:              765077
+
+Latency (ms):
+         min:                                   11.86
+         avg:                                   50.19
+         max:                                  276.58
+         95th percentile:                       81.48
+         sum:                             38399834.47
+
+Threads fairness:
+    events (avg/stddev):           11954.3281/1290.50
+    execution time (avg/stddev):   599.9974/0.01`,
+			benchmarkName:  "sysbench-settings/oltp_read_write/nodes=3/cpu=8/conc=64",
+			expectedOutput: "BenchmarkSysbenchSettings/a=oltp_read_write/nodes=3/cpu=8/conc=64\t1\t25501.03 queries/sec\t1275.05 txns/sec\t11.86 ms/min\t50.19 ms/avg\t81.48 ms/p95\t276.58 ms/max",
+			expectError:    false,
+		},
+		{
+			name:           "bad-input",
+			result:         `something`,
+			benchmarkName:  "sysbench-settings/oltp_read_write/nodes=3/cpu=8/conc=64",
+			expectedOutput: "",
+			expectError:    true,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			output, err := sysbenchToGoBench(tc.benchmarkName, tc.result)
+
+			if tc.expectError {
+				require.Error(t, err)
+			} else {
+				require.NoError(t, err)
+				require.Equal(t, tc.expectedOutput, output)
+			}
+		})
+	}
+}

pkg/sql/tests/sysbench_test.go

@@ -984,57 +984,87 @@ func (f doneFn) Stop(b testing.TB) {
 }
 
 func startAllocsProfile(b testing.TB) doneFn {
-	out := benchmemFile(b)
-	if out == "" {
-		return func(tb testing.TB) {}
+	outAllocs := testProfileFile(b, "memprofile")
+	diffAllocs := diffProfile(b, func() []byte {
+		if outAllocs == "" {
+			return nil
+		}
+		p := runtimepprof.Lookup("allocs")
+		var buf bytes.Buffer
+
+		runtime.GC()
+		require.NoError(b, p.WriteTo(&buf, 0))
+
+		return buf.Bytes()
+	})
+
+	outMutex := testProfileFile(b, "mutexprofile")
+	diffMutex := diffProfile(b, func() []byte {
+		if outMutex == "" {
+			return nil
+		}
+		p := runtimepprof.Lookup("mutex")
+		var buf bytes.Buffer
+		require.NoError(b, p.WriteTo(&buf, 0))
+		return buf.Bytes()
+	})
+
+	return func(b testing.TB) {
+		if sl := diffAllocs(b); len(sl) > 0 {
+			require.NoError(b, os.WriteFile(outAllocs, sl, 0644))
+		}
+		if sl := diffMutex(b); len(sl) > 0 {
+			require.NoError(b, os.WriteFile(outMutex, sl, 0644))
+		}
 	}
+}
 
+func diffProfile(b testing.TB, take func() []byte) func(testing.TB) []byte {
 	// The below is essentially cribbed from pprof.go in net/http/pprof.
-	p := runtimepprof.Lookup("allocs")
-	var buf bytes.Buffer
-	runtime.GC()
-	require.NoError(b, p.WriteTo(&buf, 0))
-	pBase, err := profile.ParseData(buf.Bytes())
+
+	baseBytes := take()
+	if baseBytes == nil {
+		return func(tb testing.TB) []byte { return nil }
+	}
+	pBase, err := profile.ParseData(baseBytes)
 	require.NoError(b, err)
 
-	return func(b testing.TB) {
-		runtime.GC()
-		var buf bytes.Buffer
-		require.NoError(b, p.WriteTo(&buf, 0))
-		pNew, err := profile.ParseData(buf.Bytes())
+	return func(b testing.TB) []byte {
+		pNew, err := profile.ParseData(take())
 		require.NoError(b, err)
 		pBase.Scale(-1)
 		pMerged, err := profile.Merge([]*profile.Profile{pBase, pNew})
 		require.NoError(b, err)
 		pMerged.TimeNanos = pNew.TimeNanos
 		pMerged.DurationNanos = pNew.TimeNanos - pBase.TimeNanos
 
-		buf = bytes.Buffer{}
+		buf := bytes.Buffer{}
 		require.NoError(b, pMerged.Write(&buf))
-		require.NoError(b, os.WriteFile(out, buf.Bytes(), 0644))
+		return buf.Bytes()
 	}
 }
 
-// If -test.benchmem is passed, also write a base alloc profile when the
-// setup is done. This can be used via `pprof -base` to show only the
-// allocs during run (excluding the setup).
+// If -test.<something> is passed for the flag corresponding to the given
+// profile, also write a base profile when the setup is done. This can be used
+// via `pprof -base` to show only the samples from during run (excluding the
+// setup).
 //
-// The file name for the base profile will be derived from -test.memprofile, and
+// The file name for the base profile will be derived from -test.<flag>, and
 // will contain it as a prefix (mod the file extension).
-func benchmemFile(b testing.TB) string {
+func testProfileFile(b testing.TB, flagWithoutTestPrefix string) string {
 	b.Helper()
-	var benchMemFile string
+	var flagFile string
 	var outputDir string
-	require.NoError(b, sniffarg.DoEnv("test.memprofile", &benchMemFile))
+	require.NoError(b, sniffarg.DoEnv("test."+flagWithoutTestPrefix, &flagFile))
 	require.NoError(b, sniffarg.DoEnv("test.outputdir", &outputDir))
 
-	if benchMemFile == "" {
+	if flagFile == "" {
 		return ""
 	}
 
 	saniRE := regexp.MustCompile(`\W+`)
 	saniName := saniRE.ReplaceAllString(strings.TrimPrefix(b.Name(), "Benchmark"), "_")
-	dest := strings.Replace(benchMemFile, ".", "_"+saniName+".", 1)
+	dest := strings.Replace(flagFile, ".", "_"+saniName+".", 1)
 	if outputDir != "" {
 		dest = filepath.Join(outputDir, dest)
 	}