drt: add configurable downtime for node kill operations

Previously, node kill operations would immediately attempt
to restart the killed node during the cleanup phase.
This was inadequate because it didn’t allow tests to simulate
real-world scenarios where a node remains offline for an extended
period.
This patch introduces a configurable downtime parameter to node-kill
operations. The `cleanupNodeKill` struct now includes a `downtime`
field, and the `Cleanup` method waits for the specified duration before
restarting the node. Several operation variants were registered with
different combinations of signal type, drain behavior, and downtime
durations (e.g., 1m, 10m, 1h, 5h).

Epic: none
Fixes: #138573, #138574, #138575, #138576
Release note: None

Author: Vidit Bhat

pkg/cmd/roachtest/operations/node_kill.go

@@ -20,11 +20,19 @@ import (
 )
 
 type cleanupNodeKill struct {
-	nodes option.NodeListOption
+	nodes    option.NodeListOption
+	downtime time.Duration
 }
 
 func (cl *cleanupNodeKill) Cleanup(ctx context.Context, o operation.Operation, c cluster.Cluster) {
-	// We might need to restart the node if it isn't live.
+	o.Status(fmt.Sprintf("waiting for %s before restarting node %s", cl.downtime, cl.nodes))
+	select {
+	case <-time.After(cl.downtime):
+	case <-ctx.Done():
+		o.Status("cleanup context cancelled during wait")
+		return
+	}
+
 	db, err := c.ConnE(ctx, o.L(), cl.nodes[0])
 	if err != nil {
 		err = c.RunE(ctx, option.WithNodes(cl.nodes), "./cockroach.sh")
@@ -48,15 +56,20 @@ func (cl *cleanupNodeKill) Cleanup(ctx context.Context, o operation.Operation, c
 }
 
 func nodeKillRunner(
-	signal int, drain bool,
+	signal int, drain bool, downtime time.Duration,
 ) func(ctx context.Context, o operation.Operation, c cluster.Cluster) registry.OperationCleanup {
 	return func(ctx context.Context, o operation.Operation, c cluster.Cluster) registry.OperationCleanup {
-		return runNodeKill(ctx, o, c, signal, drain)
+		return runNodeKill(ctx, o, c, signal, drain, downtime)
 	}
 }
 
 func runNodeKill(
-	ctx context.Context, o operation.Operation, c cluster.Cluster, signal int, drain bool,
+	ctx context.Context,
+	o operation.Operation,
+	c cluster.Cluster,
+	signal int,
+	drain bool,
+	downtime time.Duration,
 ) registry.OperationCleanup {
 	rng, _ := randutil.NewPseudoRand()
 	node := c.All().SeededRandNode(rng)
@@ -88,46 +101,55 @@ func runNodeKill(
 		time.Sleep(1 * time.Second)
 	}
 
-	return &cleanupNodeKill{
-		nodes: node,
-	}
+	// Schedule the cleanup instead of returning it
+	go func() {
+		cleanup := cleanupNodeKill{
+			nodes:    node,
+			downtime: downtime,
+		}
+		cleanup.Cleanup(ctx, o, c)
+	}()
+
+	// return nil to avoid the hardcoded 5s + random [0s, 24h] wait
+	return nil
 }
 
 func registerNodeKill(r registry.Registry) {
-	r.AddOperation(registry.OperationSpec{
-		Name:               "node-kill/sigkill/drain=true",
-		Owner:              registry.OwnerServer,
-		Timeout:            15 * time.Minute,
-		CompatibleClouds:   registry.AllClouds,
-		CanRunConcurrently: registry.OperationCannotRunConcurrently,
-		Dependencies:       []registry.OperationDependency{registry.OperationRequiresZeroUnderreplicatedRanges},
-		Run:                nodeKillRunner(9 /* signal */, true /* drain */),
-	})
-	r.AddOperation(registry.OperationSpec{
-		Name:               "node-kill/sigkill/drain=false",
-		Owner:              registry.OwnerServer,
-		Timeout:            10 * time.Minute,
-		CompatibleClouds:   registry.AllClouds,
-		CanRunConcurrently: registry.OperationCannotRunConcurrently,
-		Dependencies:       []registry.OperationDependency{registry.OperationRequiresZeroUnderreplicatedRanges},
-		Run:                nodeKillRunner(9 /* signal */, false /* drain */),
-	})
-	r.AddOperation(registry.OperationSpec{
-		Name:               "node-kill/sigterm/drain=true",
-		Owner:              registry.OwnerServer,
-		Timeout:            15 * time.Minute,
-		CompatibleClouds:   registry.AllClouds,
-		CanRunConcurrently: registry.OperationCannotRunConcurrently,
-		Dependencies:       []registry.OperationDependency{registry.OperationRequiresZeroUnderreplicatedRanges},
-		Run:                nodeKillRunner(15 /* signal */, true /* drain */),
-	})
-	r.AddOperation(registry.OperationSpec{
-		Name:               "node-kill/sigterm/drain=false",
-		Owner:              registry.OwnerServer,
-		Timeout:            10 * time.Minute,
-		CompatibleClouds:   registry.AllClouds,
-		CanRunConcurrently: registry.OperationCannotRunConcurrently,
-		Dependencies:       []registry.OperationDependency{registry.OperationRequiresZeroUnderreplicatedRanges},
-		Run:                nodeKillRunner(15 /* signal */, false /* drain */),
-	})
+	for _, spec := range []struct {
+		name     string
+		signal   int
+		drain    bool
+		downtime time.Duration
+		timeout  time.Duration
+	}{
+		{"node-kill/sigkill/drain=true/downtime=1m", 9, true, 1 * time.Minute, 20 * time.Minute},
+		{"node-kill/sigkill/drain=true/downtime=10m", 9, true, 10 * time.Minute, 25 * time.Minute},
+		{"node-kill/sigkill/drain=true/downtime=1h", 9, true, 1 * time.Hour, 2 * time.Hour},
+		{"node-kill/sigkill/drain=true/downtime=5h", 9, true, 5 * time.Hour, 6 * time.Hour},
+
+		{"node-kill/sigkill/drain=false/downtime=1m", 9, false, 1 * time.Minute, 20 * time.Minute},
+		{"node-kill/sigkill/drain=false/downtime=10m", 9, false, 10 * time.Minute, 25 * time.Minute},
+		{"node-kill/sigkill/drain=false/downtime=1h", 9, false, 1 * time.Hour, 2 * time.Hour},
+		{"node-kill/sigkill/drain=false/downtime=5h", 9, true, 5 * time.Hour, 6 * time.Hour},
+
+		{"node-kill/sigterm/drain=true/downtime=1m", 15, true, 1 * time.Minute, 20 * time.Minute},
+		{"node-kill/sigterm/drain=true/downtime=10m", 15, true, 10 * time.Minute, 25 * time.Minute},
+		{"node-kill/sigterm/drain=true/downtime=1h", 15, true, 1 * time.Hour, 2 * time.Hour},
+		{"node-kill/sigterm/drain=true/downtime=5h", 15, true, 5 * time.Hour, 6 * time.Hour},
+
+		{"node-kill/sigterm/drain=false/downtime=1m", 15, false, 1 * time.Minute, 20 * time.Minute},
+		{"node-kill/sigterm/drain=false/downtime=10m", 15, false, 10 * time.Minute, 25 * time.Minute},
+		{"node-kill/sigterm/drain=false/downtime=1h", 15, false, 1 * time.Hour, 2 * time.Hour},
+		{"node-kill/sigterm/drain=false/downtime=5h", 15, true, 5 * time.Hour, 6 * time.Hour},
+	} {
+		r.AddOperation(registry.OperationSpec{
+			Name:               spec.name,
+			Owner:              registry.OwnerServer,
+			Timeout:            spec.timeout,
+			CompatibleClouds:   registry.AllClouds,
+			CanRunConcurrently: registry.OperationCannotRunConcurrently,
+			Dependencies:       []registry.OperationDependency{registry.OperationRequiresZeroUnderreplicatedRanges},
+			Run:                nodeKillRunner(spec.signal, spec.drain, spec.downtime),
+		})
+	}
 }