roachtest: monitor nodes in mixedversion

This updates the `mixedversion` runner to monitor the status of the
cockroach processes on the cluster during a mixed-version test. This
allows the test to fail immediately if a node crashes unexpectedly.

Previously, if a node crashed, the test would only fail if the crash
resulted in an assertion failure. There is a chance that this would
not happen because the framework could decide to restart the node
shortly after its crash.

With this change, an unexpected node death causes the test to fail
immediately. This commit also exposes the ability for test authors to
use `ExpectDeath()` in their tests, in case the test performs its own
restarts or chaos events.

Epic: CRDB-19321

Release note: None

Author: Renato Costa

pkg/cmd/roachtest/monitor.go

@@ -31,11 +31,13 @@ type monitorImpl struct {
 		Failed() bool
 		WorkerStatus(...interface{})
 	}
-	l         *logger.Logger
-	nodes     string
-	ctx       context.Context
-	cancel    func()
-	g         *errgroup.Group
+	l      *logger.Logger
+	nodes  string
+	ctx    context.Context
+	cancel func()
+	g      *errgroup.Group
+
+	numTasks  int32 // atomically
 	expDeaths int32 // atomically
 }
 
@@ -79,6 +81,8 @@ func (m *monitorImpl) ResetDeaths() {
 var errTestFatal = errors.New("t.Fatal() was called")
 
 func (m *monitorImpl) Go(fn func(context.Context) error) {
+	atomic.AddInt32(&m.numTasks, 1)
+
 	m.g.Go(func() (err error) {
 		defer func() {
 			r := recover()
@@ -170,15 +174,21 @@ func (m *monitorImpl) wait() error {
 	}
 
 	// 1. The first goroutine waits for the worker errgroup to exit.
+	// Note that this only happens if the caller created at least one
+	// task for the monitor. This check enables the roachtest monitor to
+	// be used in cases where we just want to monitor events in the
+	// cluster without running any background tasks through the monitor.
 	var wg sync.WaitGroup
-	wg.Add(1)
-	go func() {
-		defer func() {
-			m.cancel()
-			wg.Done()
+	if atomic.LoadInt32(&m.numTasks) > 0 {
+		wg.Add(1)
+		go func() {
+			defer func() {
+				m.cancel()
+				wg.Done()
+			}()
+			setErr(errors.Wrap(m.g.Wait(), "function passed to monitor.Go failed"))
 		}()
-		setErr(errors.Wrap(m.g.Wait(), "function passed to monitor.Go failed"))
-	}()
+	}
 
 	// 2. The second goroutine reads from the monitoring channel, watching for any
 	// unexpected death events.

pkg/cmd/roachtest/roachtestutil/mixedversion/mixedversion.go

@@ -653,6 +653,7 @@ func (s restartWithNewBinaryStep) Description() string {
 func (s restartWithNewBinaryStep) Run(
 	ctx context.Context, l *logger.Logger, c cluster.Cluster, h *Helper,
 ) error {
+	h.ExpectDeath()
 	return clusterupgrade.RestartNodesWithNewBinary(
 		ctx,
 		s.rt,