roachtest: update reset VM smoke test

The reset VM failure now has the option to stop the processes before
restarting the cluster.

This change updates the test to exercise the new functionality.

Epic: None
Release note: None

Author: herkolategan

pkg/cmd/roachtest/tests/failure_injection.go

@@ -789,46 +789,51 @@ var processKillTests = func(c cluster.Cluster) []failureSmokeTest {
 	return tests
 }
 
-var resetVMTests = func(c cluster.Cluster) failureSmokeTest {
+var resetVMTests = func(c cluster.Cluster) []failureSmokeTest {
 	rng, _ := randutil.NewPseudoRand()
 	rebootedNode := c.CRDBNodes().SeededRandNode(rng)
-	return failureSmokeTest{
-		testName:    failures.ResetVMFailureName,
-		failureName: failures.ResetVMFailureName,
-		args: failures.ResetVMArgs{
-			Nodes: rebootedNode.InstallNodes(),
-		},
-		validateFailure: func(ctx context.Context, l *logger.Logger, c cluster.Cluster, f *failures.Failer) error {
-			// Check that we aren't able to establish a SQL connection to the rebooted node.
-			// waitForFailureToPropagate already does a similar check, but we do it here
-			// to satisfy the smoke test framework since this is a fairly simple failure
-			// mode with less to validate.
-			return testutils.SucceedsSoonError(func() error {
-				if ctx.Err() != nil {
-					return ctx.Err()
-				}
+	var tests []failureSmokeTest
+	for _, stopCluster := range []bool{true, false} {
+		tests = append(tests, failureSmokeTest{
+			testName:    fmt.Sprintf("%s/StopCluster=%t", failures.ResetVMFailureName, stopCluster),
+			failureName: failures.ResetVMFailureName,
+			args: failures.ResetVMArgs{
+				Nodes:         rebootedNode.InstallNodes(),
+				StopProcesses: stopCluster,
+			},
+			validateFailure: func(ctx context.Context, l *logger.Logger, c cluster.Cluster, f *failures.Failer) error {
+				// Check that we aren't able to establish a SQL connection to the rebooted node.
+				// waitForFailureToPropagate already does a similar check, but we do it here
+				// to satisfy the smoke test framework since this is a fairly simple failure
+				// mode with less to validate.
+				return testutils.SucceedsSoonError(func() error {
+					if ctx.Err() != nil {
+						return ctx.Err()
+					}
 
-				killedDB, err := c.ConnE(ctx, l, rebootedNode[0])
-				if err == nil {
-					defer killedDB.Close()
-					if err := killedDB.Ping(); err == nil {
-						return errors.Errorf("expected node %d to be dead, but it is alive", rebootedNode)
+					killedDB, err := c.ConnE(ctx, l, rebootedNode[0])
+					if err == nil {
+						defer killedDB.Close()
+						if err := killedDB.Ping(); err == nil {
+							return errors.Errorf("expected node %d to be dead, but it is alive", rebootedNode)
+						} else {
+							l.Printf("failed to connect to node %d: %v", rebootedNode, err)
+						}
 					} else {
-						l.Printf("failed to connect to node %d: %v", rebootedNode, err)
+						l.Printf("unable to establish SQL connection to node %d", rebootedNode)
 					}
-				} else {
-					l.Printf("unable to establish SQL connection to node %d", rebootedNode)
-				}
+					return nil
+				})
+			},
+			validateRecover: func(ctx context.Context, l *logger.Logger, c cluster.Cluster, f *failures.Failer) error {
 				return nil
-			})
-		},
-		validateRecover: func(ctx context.Context, l *logger.Logger, c cluster.Cluster, f *failures.Failer) error {
-			return nil
-		},
-		workload: func(ctx context.Context, c cluster.Cluster, args ...string) error {
-			return defaultFailureSmokeTestWorkload(ctx, c, "--tolerate-errors")
-		},
+			},
+			workload: func(ctx context.Context, c cluster.Cluster, args ...string) error {
+				return defaultFailureSmokeTestWorkload(ctx, c, "--tolerate-errors")
+			},
+		})
 	}
+	return tests
 }
 
 func defaultFailureSmokeTestWorkload(ctx context.Context, c cluster.Cluster, args ...string) error {
@@ -873,11 +878,11 @@ func runFailureSmokeTest(ctx context.Context, t test.Test, c cluster.Cluster, no
 		asymmetricOutgoingNetworkPartitionTest(c),
 		latencyTest(c),
 		dmsetupDiskStallTest(c),
-		resetVMTests(c),
 		cgroupStallLogsTest(c),
 	}
 	failureSmokeTests = append(failureSmokeTests, cgroupsDiskStallTests(c)...)
 	failureSmokeTests = append(failureSmokeTests, processKillTests(c)...)
+	failureSmokeTests = append(failureSmokeTests, resetVMTests(c)...)
 
 	// Randomize the order of the tests in case any of the failures have unexpected side
 	// effects that may mask failures, e.g. a cgroups disk stall isn't properly recovered