Merge pull request #1315 from nebius/release-1.21.10/fix-wait-controller

Remove controller service DNS check from worker wait logic to support failover scenarios

Author: theyoprst

images/worker/wait-for-controller.sh

@@ -1,33 +1,13 @@
 #!/bin/bash
 
 echo "Waiting for Slurm controller to be ready..."
-controller_service="${CONTROLLER_SERVICE}"
-controller_port="${CONTROLLER_PORT:-6817}"  # Default to 6817 if not set
 max_attempts=60
 attempt=0
 
 # Create symlink to slurm configs (same as worker entrypoint)
 echo "Creating symlink to slurm configs..."
 rm -rf /etc/slurm && ln -s /mnt/jail/etc/slurm /etc/slurm
 
-# Wait for controller service to be resolvable via DNS
-echo "Checking controller service DNS resolution..."
-attempt=0
-while [ $attempt -lt $max_attempts ]; do
-	if timeout 1 bash -c "</dev/tcp/$controller_service/$controller_port" >/dev/null 2>&1; then
-		echo "Controller service is reachable on port $controller_port"
-		break
-	fi
-	echo "Attempt $((attempt + 1))/$max_attempts: Waiting for controller service TCP port $controller_port..."
-	attempt=$((attempt + 1))
-	sleep 5
-done
-
-if ! timeout 1 bash -c "</dev/tcp/$controller_service/$controller_port" >/dev/null 2>&1; then
-	echo "ERROR: Controller service TCP port $controller_port not reachable after $max_attempts attempts"
-	exit 1
-fi
-
 # Now try to ping the controller using scontrol
 echo "Checking slurmctld readiness..."
 attempt=0
@@ -38,10 +18,10 @@ while [ $attempt -lt $max_attempts ]; do
 	echo "Running: scontrol ping"
 	if scontrol_output=$(scontrol ping 2>&1); then
 		echo "Controller is ready!"
-		echo "scontrol ping output: $scontrol_output"
+		echo -e "scontrol ping output:\n$scontrol_output"
 		exit 0
 	else
-		echo "scontrol ping failed with output: $scontrol_output"
+		echo -e "scontrol ping failed with output:\n$scontrol_output"
 	fi
 
 	attempt=$((attempt + 1))

internal/controller/clustercontroller/worker.go

@@ -234,7 +234,6 @@ func (r SlurmClusterReconciler) ReconcileWorkers(
 						&clusterValues.NodeWorker,
 						clusterValues.SlurmTopologyConfigMapRefName,
 						clusterValues.WorkerFeatures,
-						clusterValues.NodeController.ContainerSlurmctld.Port,
 					)
 					if err != nil {
 						stepLogger.Error(err, "Failed to render")

internal/render/worker/container.go

@@ -47,24 +47,15 @@ func renderContainerToolkitValidation(container *values.Container) corev1.Contai
 }
 
 // RenderContainerWaitForController renders init [corev1.Container] that waits for controller readiness
-func RenderContainerWaitForController(container *values.Container, controllerPort int32) corev1.Container {
+func RenderContainerWaitForController(container *values.Container) corev1.Container {
 	return corev1.Container{
 		Name:            consts.ContainerNameWaitForController,
 		Image:           container.Image,
 		ImagePullPolicy: container.ImagePullPolicy,
 		Command: []string{
 			"/opt/bin/slurm/wait-for-controller.sh",
 		},
-		Env: []corev1.EnvVar{
-			{
-				Name:  "CONTROLLER_SERVICE",
-				Value: fmt.Sprintf("%s-%d", consts.ComponentTypeController, 0),
-			},
-			{
-				Name:  "CONTROLLER_PORT",
-				Value: strconv.FormatInt(int64(controllerPort), 10),
-			},
-		},
+		Env: []corev1.EnvVar{},
 		VolumeMounts: []corev1.VolumeMount{
 			common.RenderVolumeMountJail(),
 			common.RenderVolumeMountMungeSocket(),

internal/render/worker/statefulset.go

@@ -30,7 +30,6 @@ func RenderStatefulSet(
 	worker *values.SlurmWorker,
 	slurmTopologyConfigMapRefName string,
 	workerFeatures []slurmv1.WorkerFeature,
-	controllerPort int32,
 ) (kruisev1b1.StatefulSet, error) {
 	labels := common.RenderLabels(consts.ComponentTypeWorker, clusterName)
 	matchLabels := common.RenderMatchLabels(consts.ComponentTypeWorker, clusterName)
@@ -53,7 +52,7 @@ func RenderStatefulSet(
 		common.RenderContainerMunge(&worker.ContainerMunge),
 	}
 	if worker.WaitForController != nil && *worker.WaitForController {
-		initContainers = append(initContainers, RenderContainerWaitForController(&worker.ContainerSlurmd, controllerPort))
+		initContainers = append(initContainers, RenderContainerWaitForController(&worker.ContainerSlurmd))
 	}
 	if clusterType == consts.ClusterTypeGPU {
 		initContainers = append(initContainers, renderContainerToolkitValidation(&worker.ContainerToolkitValidation))

internal/render/worker/statefulset_test.go

@@ -52,7 +52,6 @@ func Test_RenderStatefulSet(t *testing.T) {
 			},
 		},
 	}
-	controllerPort := int32(6817)
 
 	createWorker := func() *values.SlurmWorker {
 		return &values.SlurmWorker{
@@ -131,7 +130,7 @@ func Test_RenderStatefulSet(t *testing.T) {
 	for _, tt := range tests {
 		t.Run(tt.name, func(t *testing.T) {
 			result, err := worker.RenderStatefulSet(
-				testNamespace, testCluster, tt.clusterType, nodeFilter, tt.secrets, volumeSource, tt.worker, testTopologyConfig, nil, controllerPort,
+				testNamespace, testCluster, tt.clusterType, nodeFilter, tt.secrets, volumeSource, tt.worker, testTopologyConfig, nil,
 			)
 			assert.NoError(t, err)
 
@@ -169,25 +168,20 @@ func Test_RenderStatefulSet(t *testing.T) {
 }
 
 func Test_RenderContainerWaitForController(t *testing.T) {
-	controllerPort := int32(6817)
 	container := &values.Container{
 		NodeContainer: slurmv1.NodeContainer{
 			Image:           "test-image",
 			ImagePullPolicy: corev1.PullIfNotPresent,
 		},
 	}
 
-	result := worker.RenderContainerWaitForController(container, controllerPort)
+	result := worker.RenderContainerWaitForController(container)
 
 	assert.Equal(t, consts.ContainerNameWaitForController, result.Name)
 	assert.Equal(t, container.Image, result.Image)
 	assert.Equal(t, container.ImagePullPolicy, result.ImagePullPolicy)
 	assert.Equal(t, []string{"/opt/bin/slurm/wait-for-controller.sh"}, result.Command)
-	assert.Equal(t, 2, len(result.Env))
-	assert.Equal(t, "CONTROLLER_SERVICE", result.Env[0].Name)
-	assert.Equal(t, "controller-0", result.Env[0].Value)
-	assert.Equal(t, "CONTROLLER_PORT", result.Env[1].Name)
-	assert.Equal(t, "6817", result.Env[1].Value)
+	assert.Equal(t, 0, len(result.Env))
 	assert.Equal(t, 2, len(result.VolumeMounts))
 
 	// Verify exact volume mount values and no unexpected mounts