Merge #143738

143738: roachprod-microbench: post github issues r=DarrylWong,srosenberg a=herkolategan

This PR adds functionality to automatically post GitHub issues for microbenchmark failures in CI (teamcity).

It will only post failures for executions from the master branch. Typically two binaries are interleaved to produce a comparison, but with this change the binary from master (experiment) will be executed first, if it fails it will cancel all further executions of the benchmark and post the failure to github.

There is one caveat regarding the posted issues. We retrieve stdout and stderr separately, so the log (on a github issue) will be a concatenation of the two.

Failures will be labeled with `O-microbench` as well as `C-test-failure` and `release-blocker`
A timeout (> 20 minutes for one iteration) will also count as a failure.

Epic: None
Release note: None

Co-authored-by: Herko Lategan <[email protected]>

Author: craig[bot]

build/teamcity/cockroach/nightlies/microbenchmark_weekly.sh

@@ -139,6 +139,13 @@ for sha in "${build_sha_arr[@]}"; do
   ./bin/roachprod-microbench stage --quiet "$ROACHPROD_CLUSTER" "gs://$BENCH_BUCKET/builds/$archive_name" "$remote_dir/$sha"
 done
 
+# Post issues to github for triggered builds (triggered builds are always on master)
+if [ -n "${TRIGGERED_BUILD:-}" ]; then
+  GITHUB_BRANCH="master"
+  GITHUB_SHA="${build_sha_arr[0]}"
+  GITHUB_BINARY="experiment"
+fi
+
 # Execute microbenchmarks
 ./bin/roachprod-microbench run "$ROACHPROD_CLUSTER" \
   --binaries experiment="$remote_dir/${build_sha_arr[0]}" \
@@ -149,7 +156,8 @@ done
   ${BENCH_TIMEOUT:+--timeout="$BENCH_TIMEOUT"} \
   ${BENCH_EXCLUDE:+--exclude="$BENCH_EXCLUDE"} \
   ${BENCH_IGNORE_PACKAGES:+--ignore-package="$BENCH_IGNORE_PACKAGES"} \
-  --quiet \
+  ${TRIGGERED_BUILD:+--post-issues} \
+  --quiet \  
   -- "$TEST_ARGS" \
   || exit_status=$?
 

pkg/cmd/bazci/githubpost/githubpost.go

@@ -758,3 +758,12 @@ func postGeneralFailureImpl(logs string, fileIssue func(context.Context, Failure
 	}
 
 }
+
+// MicrobenchmarkFailure creates a Failure struct for a microbenchmark failure.
+func MicrobenchmarkFailure(packageName string, benchmarkName string, logs string) Failure {
+	return Failure{
+		packageName: packageName,
+		testName:    benchmarkName,
+		testMessage: logs,
+	}
+}

pkg/cmd/roachprod-microbench/BUILD.bazel

@@ -8,6 +8,7 @@ go_library(
         "compress.go",
         "executor.go",
         "export.go",
+        "github.go",
         "main.go",
         "metadata.go",
         "report.go",
@@ -18,6 +19,8 @@ go_library(
     importpath = "github.com/cockroachdb/cockroach/pkg/cmd/roachprod-microbench",
     visibility = ["//visibility:private"],
     deps = [
+        "//pkg/cmd/bazci/githubpost",
+        "//pkg/cmd/bazci/githubpost/issues",
         "//pkg/cmd/roachprod-microbench/cluster",
         "//pkg/cmd/roachprod-microbench/google",
         "//pkg/cmd/roachprod-microbench/model",
@@ -54,15 +57,19 @@ go_test(
         "compare_test.go",
         "executor_test.go",
         "export_test.go",
+        "github_test.go",
     ],
     data = glob(["testdata/**"]),
     embed = [":roachprod-microbench_lib"],
     deps = [
+        "//pkg/cmd/bazci/githubpost/issues",
+        "//pkg/cmd/roachprod-microbench/cluster",
         "//pkg/cmd/roachprod-microbench/model",
         "//pkg/cmd/roachprod-microbench/parser",
         "//pkg/testutils/datapathutils",
         "//pkg/util/timeutil",
         "@com_github_cockroachdb_datadriven//:datadriven",
+        "@com_github_cockroachdb_errors//:errors",
         "@com_github_stretchr_testify//require",
         "@org_golang_x_exp//maps",
     ],

pkg/cmd/roachprod-microbench/executor.go

@@ -44,6 +44,7 @@ type executorConfig struct {
 	affinity          bool
 	quiet             bool
 	recoverable       bool
+	postIssues        bool
 }
 
 type executor struct {
@@ -52,6 +53,13 @@ type executor struct {
 	ignorePackages         map[string]struct{}
 	runOptions             install.RunOptions
 	log                    *logger.Logger
+	postConfig             postConfig
+}
+
+type postConfig struct {
+	branch    string
+	binary    string
+	commitSHA string
 }
 
 type benchmark struct {
@@ -103,6 +111,18 @@ func newExecutor(config executorConfig) (*executor, error) {
 		return nil, errors.New("iterations must be greater than 0")
 	}
 
+	var pc postConfig
+	if config.postIssues {
+		pc = postConfig{
+			branch:    os.Getenv("GITHUB_BRANCH"),
+			binary:    os.Getenv("GITHUB_BINARY"),
+			commitSHA: os.Getenv("GITHUB_SHA"),
+		}
+		if pc.branch == "" || pc.binary == "" || pc.commitSHA == "" {
+			return nil, errors.New("GITHUB_BRANCH, GITHUB_BINARY, and GITHUB_SHA environment variables must be set when post-issues is enabled")
+		}
+	}
+
 	roachprodConfig.Quiet = config.quiet
 	timestamp := timeutil.Now()
 	l := InitLogger(filepath.Join(config.outputDir, fmt.Sprintf("roachprod-microbench-%s.log", timestamp.Format(util.TimeFormat))))
@@ -114,6 +134,7 @@ func newExecutor(config executorConfig) (*executor, error) {
 		ignorePackages:         ignorePackages,
 		runOptions:             runOptions,
 		log:                    l,
+		postConfig:             pc,
 	}, nil
 }
 
@@ -265,6 +286,21 @@ func (e *executor) generateBenchmarkCommands(
 	binaryKeys := maps.Keys(e.binaries)
 	sort.Strings(binaryKeys)
 
+	// If post issues is enabled, move the post config binary key to the front
+	// of the binary keys list to ensure it runs first. Since we might only run
+	// one iteration before cancelling the other iterations, we want to report
+	// the failure as soon as possible.
+	if e.postIssues {
+		for i, key := range binaryKeys {
+			if key == e.postConfig.binary {
+				// Move the key to front by removing it and inserting at index 0
+				copy(binaryKeys[1:i+1], binaryKeys[0:i])
+				binaryKeys[0] = e.postConfig.binary
+				break
+			}
+		}
+	}
+
 	// Generate the commands for each benchmark binary.
 	for _, bench := range benchmarks {
 		runCommand := fmt.Sprintf("./run.sh %s -test.benchmem -test.bench=^%s$ -test.run=^$ -test.v",
@@ -283,6 +319,7 @@ func (e *executor) generateBenchmarkCommands(
 			command := cluster.RemoteCommand{
 				Args:     []string{"sh", "-c", shellCommand},
 				Metadata: benchmarkKey{bench, key},
+				GroupID:  fmt.Sprintf("%s/%s", bench.pkg, bench.name),
 			}
 			benchmarkCommands = append(benchmarkCommands, command)
 		}
@@ -314,6 +351,7 @@ func (e *executor) generateBenchmarkCommands(
 // corresponding microbenchmark. When running in lenient mode errors will not
 // fail the execution, and will still be logged to the aforementioned logs.
 func (e *executor) executeBenchmarks() error {
+	var executorError error
 
 	// Remote execution Logging is captured and saved to appropriate log files and
 	// the main logger is used for orchestration logging only. Therefore, we use a
@@ -387,7 +425,6 @@ func (e *executor) executeBenchmarks() error {
 	}
 
 	// Execute commands.
-	errorCount := 0
 	logIndex := 0
 	missingBenchmarks := make(map[benchmark]int, 0)
 	failedBenchmarks := make(map[benchmark]int, 0)
@@ -410,14 +447,25 @@ func (e *executor) executeBenchmarks() error {
 				fmt.Println()
 			}
 			tag := fmt.Sprintf("%d", logIndex)
+			timeout := false
 			if response.ExitStatus == 124 || response.ExitStatus == 137 {
 				tag = fmt.Sprintf("%d-timeout", logIndex)
+				timeout = true
 			}
 			err = report.writeBenchmarkErrorLogs(response, tag)
 			if err != nil {
 				e.log.Errorf("Failed to write error logs - %v", err)
 			}
-			errorCount++
+
+			if e.postIssues && benchmarkResponse.key == e.postConfig.binary {
+				artifactsDir := fmt.Sprintf("%s/%s", e.outputDir, benchmarkResponse.key)
+				formatter, req := createBenchmarkPostRequest(artifactsDir, response, timeout)
+				err = postBenchmarkIssue(context.Background(), e.log, formatter, req)
+				if err != nil {
+					e.log.Errorf("Failed to post benchmark issue - %v", err)
+					executorError = errors.CombineErrors(executorError, errors.Wrap(err, "failed to post benchmark issue"))
+				}
+			}
 			logIndex++
 		}
 		if _, writeErr := report.analyticsOutput[benchmarkResponse.pkg].WriteString(
@@ -456,7 +504,7 @@ func (e *executor) executeBenchmarks() error {
 	for res, count := range missingBenchmarks {
 		e.log.Errorf("Missing benchmark: %s/%s in %d iterations", res.pkg, res.name, count)
 	}
-
 	e.log.Printf("Completed benchmarks, results located at %s", e.outputDir)
-	return nil
+
+	return executorError
 }

pkg/cmd/roachprod-microbench/github.go

@@ -0,0 +1,53 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+//
+
+package main
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/cmd/bazci/githubpost"
+	"github.com/cockroachdb/cockroach/pkg/cmd/bazci/githubpost/issues"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachprod-microbench/cluster"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
+)
+
+// createBenchmarkPostRequest creates a post request for a benchmark failure.
+func createBenchmarkPostRequest(
+	artifactsDir string, response cluster.RemoteResponse, timeout bool,
+) (issues.IssueFormatter, issues.PostRequest) {
+	b := response.Metadata.(benchmarkKey).benchmark
+	var combinedOutput strings.Builder
+	if timeout {
+		combinedOutput.WriteString(fmt.Sprintf("%s timed out after %s\n", b.name, response.Duration.Round(time.Second)))
+	} else {
+		combinedOutput.WriteString(response.Stdout)
+		combinedOutput.WriteString("\n")
+		combinedOutput.WriteString(response.Stderr)
+	}
+
+	f := githubpost.MicrobenchmarkFailure(
+		b.pkg,
+		b.name,
+		combinedOutput.String(),
+	)
+	formatter, req := githubpost.DefaultFormatter(context.Background(), f)
+	req.Artifacts = artifactsDir
+	req.Labels = append(req.Labels, "O-microbench")
+	return formatter, req
+}
+
+// postBenchmarkIssue posts a benchmark issue to github.
+func postBenchmarkIssue(
+	ctx context.Context, l *logger.Logger, formatter issues.IssueFormatter, req issues.PostRequest,
+) error {
+	opts := issues.DefaultOptionsFromEnv()
+	_, err := issues.Post(ctx, l, formatter, req, opts)
+	return err
+}

pkg/cmd/roachprod-microbench/github_test.go

@@ -0,0 +1,94 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+//
+
+package main
+
+import (
+	"fmt"
+	"net/url"
+	"strings"
+	"testing"
+	"time"
+
+	"github.com/cockroachdb/cockroach/pkg/cmd/bazci/githubpost/issues"
+	"github.com/cockroachdb/cockroach/pkg/cmd/roachprod-microbench/cluster"
+	"github.com/cockroachdb/cockroach/pkg/testutils/datapathutils"
+	"github.com/cockroachdb/datadriven"
+	"github.com/cockroachdb/errors"
+)
+
+func TestCreatePostRequest(t *testing.T) {
+	datadriven.Walk(t, datapathutils.TestDataPath(t, "github"), func(t *testing.T, path string) {
+		var response cluster.RemoteResponse
+		var bk benchmarkKey
+		datadriven.RunTest(t, path, func(t *testing.T, d *datadriven.TestData) string {
+			switch d.Cmd {
+			case "benchmark":
+				d.ScanArgs(t, "name", &bk.name)
+				d.ScanArgs(t, "pkg", &bk.pkg)
+				d.ScanArgs(t, "args", &response.Args)
+				return ""
+			case "stdout":
+				response.Stdout = d.Input
+				return ""
+			case "stderr":
+				response.Stderr = d.Input
+				return ""
+			case "post":
+				response.Err = errors.New("benchmark failed")
+				response.ExitStatus = 1
+				response.Duration = time.Second * 10
+				response.Metadata = bk
+				formatter, req := createBenchmarkPostRequest("", response, false)
+				str, err := formatPostRequest(formatter, req)
+				if err != nil {
+					t.Fatal(err)
+				}
+				return str
+			}
+			return ""
+		})
+	})
+}
+
+// formatPostRequest emulates the behavior of the githubpost package.
+func formatPostRequest(formatter issues.IssueFormatter, req issues.PostRequest) (string, error) {
+	// These fields can vary based on the test env so we set them to arbitrary
+	// values here.
+	req.MentionOnCreate = []string{"@test-eng"}
+	req.ProjectColumnID = 0
+
+	data := issues.TemplateData{
+		PostRequest:      req,
+		Parameters:       req.ExtraParams,
+		CondensedMessage: issues.CondensedMessage(req.Message),
+		PackageNameShort: req.PackageName,
+	}
+
+	r := &issues.Renderer{}
+	if err := formatter.Body(r, data); err != nil {
+		return "", err
+	}
+
+	var post strings.Builder
+	post.WriteString(r.String())
+
+	u, err := url.Parse("https://github.com/cockroachdb/cockroach/issues/new")
+	if err != nil {
+		return "", err
+	}
+	q := u.Query()
+	q.Add("title", formatter.Title(data))
+	q.Add("body", post.String())
+	// Adding a template parameter is required to be able to view the rendered
+	// template on GitHub, otherwise it just takes you to the template selection
+	// page.
+	q.Add("template", "none")
+	u.RawQuery = q.Encode()
+	post.WriteString(fmt.Sprintf("Rendered:\n%s", u.String()))
+
+	return post.String(), nil
+}

pkg/cmd/roachprod-microbench/main.go

@@ -116,6 +116,7 @@ func makeRunCommand() *cobra.Command {
 	cmd.Flags().BoolVar(&config.affinity, "affinity", config.affinity, "run benchmarks with each iteration's binaries having affinity to the same node, while different iterations can run on different nodes")
 	cmd.Flags().BoolVar(&config.quiet, "quiet", config.quiet, "suppress roachprod progress output")
 	cmd.Flags().BoolVar(&config.recoverable, "recoverable", config.recoverable, "VMs are able to recover from transient failures (e.g., running spot instances on a MIG in GCE)")
+	cmd.Flags().BoolVar(&config.postIssues, "post-issues", config.postIssues, "post issues to github (requires env vars for github issues to be set)")
 	return cmd
 }