failureinjection: catch runAsync panics as errors

Now that the failureinjection framework can spin up goroutines,
make sure to catch all panics as errors. This keeps it compatible
with the roachtest framework, where we don't want any uncaught
panics to take down an entire run.

Author: DarrylWong

pkg/roachprod/failureinjection/failures/BUILD.bazel

@@ -18,6 +18,7 @@ go_library(
         "//pkg/roachprod",
         "//pkg/roachprod/install",
         "//pkg/roachprod/logger",
+        "//pkg/roachprod/roachprodutil",
         "//pkg/util/retry",
         "//pkg/util/syncutil",
         "//pkg/util/sysutil",

pkg/roachprod/failureinjection/failures/failure.go

@@ -15,6 +15,7 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/roachprod"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/install"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/roachprodutil"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
 	"github.com/cockroachdb/errors"
@@ -284,10 +285,12 @@ func forEachNode(nodes install.Nodes, fn func(install.Nodes) error) error {
 	return nil
 }
 
-func runAsync(f func() error) <-chan error {
+func runAsync(ctx context.Context, l *logger.Logger, f func(context.Context) error) <-chan error {
 	errCh := make(chan error, 1)
 	go func() {
-		err := f()
+		err := roachprodutil.PanicAsError(ctx, l, func(context.Context) error {
+			return f(ctx)
+		})
 		errCh <- err
 		close(errCh)
 	}()

pkg/roachprod/failureinjection/failures/process_kill.go

@@ -87,7 +87,7 @@ func (f *ProcessKillFailure) Inject(ctx context.Context, l *logger.Logger, args
 	// WaitForFailureToPropagate. We run Stop in a goroutine to achieve this, although it
 	// does mean we will ignore all errors unless the user also calls WaitForFailureToPropagate.
 	// We make this tradeoff in order to avoid maintaining two different Stop implementations.
-	f.waitCh = runAsync(func() error {
+	f.waitCh = runAsync(ctx, l, func(ctx context.Context) error {
 		return f.c.WithNodes(nodes).Stop(ctx, l, int(signal), true, gracePeriod, label)
 	})
 	return nil

pkg/roachprod/install/BUILD.bazel

@@ -33,14 +33,14 @@ go_library(
         "//pkg/roachprod/config",
         "//pkg/roachprod/errors",
         "//pkg/roachprod/logger",
+        "//pkg/roachprod/roachprodutil",
         "//pkg/roachprod/ssh",
         "//pkg/roachprod/ui",
         "//pkg/roachprod/vm",
         "//pkg/roachprod/vm/aws",
         "//pkg/roachprod/vm/gce",
         "//pkg/roachprod/vm/local",
         "//pkg/testutils",
-        "//pkg/util/debugutil",
         "//pkg/util/intsets",
         "//pkg/util/log",
         "//pkg/util/retry",

pkg/roachprod/install/cockroach.go

@@ -25,10 +25,10 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/ccl/utilccl/licenseccl"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/config"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
+	"github.com/cockroachdb/cockroach/pkg/roachprod/roachprodutil"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/ssh"
 	"github.com/cockroachdb/cockroach/pkg/roachprod/vm/gce"
 	"github.com/cockroachdb/cockroach/pkg/testutils"
-	"github.com/cockroachdb/cockroach/pkg/util/debugutil"
 	"github.com/cockroachdb/cockroach/pkg/util/retry"
 	"github.com/cockroachdb/cockroach/pkg/util/syncutil"
 	"github.com/cockroachdb/cockroach/pkg/util/timeutil"
@@ -443,7 +443,7 @@ func (c *SyncedCluster) Start(ctx context.Context, l *logger.Logger, startOpts S
 	for _, hook := range startOpts.PreStartHooks {
 		hookCtx, cancel := context.WithTimeout(ctx, hook.Timeout)
 		l.Printf("running pre-start hook: %s", hook.Name)
-		err := panicAsError(hookCtx, l, hook.Fn)
+		err := roachprodutil.PanicAsError(hookCtx, l, hook.Fn)
 		cancel()
 		if err != nil {
 			return err
@@ -1703,23 +1703,3 @@ type PreStartHook struct {
 	Fn      func(context.Context) error
 	Timeout time.Duration
 }
-
-// logPanicToErr logs the panic stack trace and returns an error with the
-// panic message.
-func panicAsError(
-	ctx context.Context, l *logger.Logger, f func(context.Context) error,
-) (retErr error) {
-	defer func() {
-		if r := recover(); r != nil {
-			retErr = logPanicToErr(l, r)
-		}
-	}()
-	return f(ctx)
-}
-
-// logPanicToErr logs the panic stack trace and returns an error with the
-// panic message.
-func logPanicToErr(l *logger.Logger, r interface{}) error {
-	l.Printf("panic stack trace:\n%s", debugutil.Stack())
-	return fmt.Errorf("panic (stack trace above): %v", r)
-}

pkg/roachprod/roachprodutil/BUILD.bazel

@@ -2,10 +2,15 @@ load("@io_bazel_rules_go//go:def.bzl", "go_library")
 
 go_library(
     name = "roachprodutil",
-    srcs = ["identity.go"],
+    srcs = [
+        "identity.go",
+        "utils.go",
+    ],
     importpath = "github.com/cockroachdb/cockroach/pkg/roachprod/roachprodutil",
     visibility = ["//visibility:public"],
     deps = [
+        "//pkg/roachprod/logger",
+        "//pkg/util/debugutil",
         "@com_github_cockroachdb_errors//:errors",
         "@com_google_cloud_go_storage//:storage",
         "@org_golang_google_api//idtoken",

pkg/roachprod/roachprodutil/utils.go

@@ -0,0 +1,34 @@
+// Copyright 2025 The Cockroach Authors.
+//
+// Use of this software is governed by the CockroachDB Software License
+// included in the /LICENSE file.
+
+package roachprodutil
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/cockroachdb/cockroach/pkg/roachprod/logger"
+	"github.com/cockroachdb/cockroach/pkg/util/debugutil"
+)
+
+// PanicAsError logs the panic stack trace and returns an error with the
+// panic message.
+func PanicAsError(
+	ctx context.Context, l *logger.Logger, f func(context.Context) error,
+) (retErr error) {
+	defer func() {
+		if r := recover(); r != nil {
+			retErr = logPanicToErr(l, r)
+		}
+	}()
+	return f(ctx)
+}
+
+// logPanicToErr logs the panic stack trace and returns an error with the
+// panic message.
+func logPanicToErr(l *logger.Logger, r interface{}) error {
+	l.Printf("panic stack trace:\n%s", debugutil.Stack())
+	return fmt.Errorf("panic (stack trace above): %v", r)
+}