roachtest: fix test monitor error

Previously, the global test monitor would fatal on an error. This is similar to
how the original monitor worked. But the original monitor would fatal where a
panic could be recovered by the goroutine running the test. This would
immediately short circuit the test. However, the global monitor can not use
fatal since it's running in a separate goroutine which causes the panic to be
unrecoverable. This change makes the global monitor call `Errorf` instead of
`Fatalf`. The test context will now instead be cancelled.

Epic: None
Release note: None

Author: herkolategan

pkg/cmd/roachtest/test_monitor.go

@@ -11,23 +11,28 @@ import (
 	"github.com/cockroachdb/cockroach/pkg/cmd/roachtest/test"
 )
 
-var _ test.Monitor = &testMonitorImpl{}
-
 type testMonitorImpl struct {
-	*monitorImpl
+	monitor interface {
+		test.Monitor
+		WaitForNodeDeath() error
+	}
+	t test.Test
 }
 
-func newTestMonitor(ctx context.Context, t test.Test, c *clusterImpl) *testMonitorImpl {
+// newTestMonitor is a function that creates a new test monitor. It can be used for testing
+// purposes to replace the default monitor with a custom implementation.
+var newTestMonitor = func(ctx context.Context, t test.Test, c *clusterImpl) *testMonitorImpl {
 	return &testMonitorImpl{
-		monitorImpl: newMonitor(ctx, t, c, true /* expectExactProcessDeath */),
+		monitor: newMonitor(ctx, t, c, true /* expectExactProcessDeath */),
+		t:       t,
 	}
 }
 
 func (m *testMonitorImpl) start() {
 	go func() {
-		err := m.WaitForNodeDeath()
+		err := m.monitor.WaitForNodeDeath()
 		if err != nil {
-			m.t.Fatal(err)
+			m.t.Errorf("test monitor: %v", err)
 		}
 	}()
 }

pkg/cmd/roachtest/test_runner.go

@@ -1351,7 +1351,7 @@ func (r *testRunner) runTest(
 
 	t.taskManager = task.NewManager(runCtx, t.L())
 	testMonitor := newTestMonitor(runCtx, t, c)
-	t.monitor = testMonitor
+	t.monitor = testMonitor.monitor
 
 	t.mu.Lock()
 	// t.Fatal() will cancel this context.