Merge #154383

154383: roachprod: auto restart option for cockroach processes r=srosenberg a=herkolategan

Previously, if a cockroach process died, it needed to be manually restarted.
This is expected in most circumstances. But this behavior might not always be
practical, for instance, when running a long unattended benchmark.

To support this, a new start option has been added that automatically restarts
the service if a process fails (e.g., due to a disk stall). This ensures the
process comes back online quickly without manual intervention.

Epic: None
Release note: None

Co-authored-by: Herko Lategan <[email protected]>

Author: craig[bot]

pkg/cmd/roachprod/cli/flags.go

@@ -453,6 +453,8 @@ func initFlagsStartOpsForCmd(cmd *cobra.Command) {
 	cmd.Flags().BoolVar(&startOpts.EnableFluentSink,
 		"enable-fluent-sink", startOpts.EnableFluentSink,
 		"whether to enable the fluent-servers attribute in the CockroachDB logging configuration")
+	cmd.Flags().BoolVar(&startOpts.AutoRestart,
+		"auto-restart", startOpts.AutoRestart, "automatically restart cockroach processes that die")
 }
 
 func initFlagInsecureIgnoreHostKeyForCmd(cmd *cobra.Command) {

pkg/roachprod/install/cockroach.go

@@ -152,6 +152,9 @@ type StartOpts struct {
 	// initialization and sequential node starts and also reuses the previous start script.
 	IsRestart bool
 
+	// AutoRestart enables automatically restarting a process if it died.
+	AutoRestart bool
+
 	// EnableFluentSink determines whether to enable the fluent-servers attribute
 	// in the CockroachDB logging configuration.
 	EnableFluentSink bool
@@ -1004,6 +1007,7 @@ func (c *SyncedCluster) generateStartCmd(
 		NumFilesLimit:       startOpts.NumFilesLimit,
 		VirtualClusterLabel: VirtualClusterLabel(startOpts.VirtualClusterName, startOpts.SQLInstance),
 		Local:               c.IsLocal(),
+		AutoRestart:         startOpts.AutoRestart,
 	})
 }
 
@@ -1017,6 +1021,7 @@ type startTemplateData struct {
 	VirtualClusterLabel string
 	Args                []string
 	EnvVars             []string
+	AutoRestart         bool
 }
 
 type loggingTemplateData struct {

pkg/roachprod/install/scripts/start.sh

@@ -16,6 +16,10 @@ KEY_CMD=#{.KeyCmd#}
 MEMORY_MAX=#{.MemoryMax#}
 NUM_FILES_LIMIT=#{.NumFilesLimit#}
 VIRTUAL_CLUSTER_LABEL=#{.VirtualClusterLabel#}
+#{if .AutoRestart#}
+AUTO_RESTART=1
+#{end#}
+
 ARGS=(
 #{range .Args -#}
 #{shesc .#}
@@ -99,4 +103,5 @@ sudo systemd-run --unit "${VIRTUAL_CLUSTER_LABEL}" \
   -p "MemoryMax=${MEMORY_MAX}" \
   -p LimitCORE=infinity \
   -p "LimitNOFILE=${NUM_FILES_LIMIT}" \
+  ${AUTO_RESTART:+-p Restart=always -p RestartSec=5s -p StartLimitIntervalSec=60s -p StartLimitBurst=3} \
   bash "${0}" run

pkg/roachprod/install/testdata/start/start.txt

@@ -20,6 +20,8 @@ echo bar $HOME
 MEMORY_MAX=81%
 NUM_FILES_LIMIT=0
 VIRTUAL_CLUSTER_LABEL=cockroach-system
+
+
 ARGS=(
 start
 --log
@@ -103,6 +105,7 @@ sudo systemd-run --unit "${VIRTUAL_CLUSTER_LABEL}" \
   -p "MemoryMax=${MEMORY_MAX}" \
   -p LimitCORE=infinity \
   -p "LimitNOFILE=${NUM_FILES_LIMIT}" \
+  ${AUTO_RESTART:+-p Restart=always -p RestartSec=5s -p StartLimitIntervalSec=60s -p StartLimitBurst=3} \
   bash "${0}" run
 ----
 ----