Implement conductor startup/liveness probes

This makes the following changes:
- The readiness probe is remove because this is only used when the Pod
  is associated with a Service
- A startup and liveness probe is added which does the following:
  - loads the ironic config
  - loads the DB API
  - looks for the configured host in the currently online hosts
- This script achieves the following in terms of a check
  - The script can access the database with the configuration
  - The list of online conductors is filtered by updated_at, so if this
    conductor has stopped sending its heartbeat since
    [conductor]heartbeat_timeout then the probe will fail
- The startup probe is run every 2 seconds since conductor will
  generally start quickly
- The liveness probe is run every 30 seconds

Jira: OSPRH-22659

Author: steveb

internal/controller/ironicconductor_controller.go

@@ -946,6 +946,7 @@ func (r *IronicConductorReconciler) generateServiceConfigMaps(
 				"runlogwatch.sh": "/common/bin/runlogwatch.sh",
 				"pxe-init.sh":    "/common/bin/pxe-init.sh",
 				"init.sh":        "/ironicconductor/bin/init.sh",
+				"live_check":     "/ironicconductor/bin/live_check",
 			},
 			Labels: cmLabels,
 		},

internal/ironicconductor/statefulset.go

@@ -53,14 +53,14 @@ func StatefulSet(
 	runAsUser := int64(0)
 
 	livenessProbe := &corev1.Probe{
-		TimeoutSeconds:      5,
-		PeriodSeconds:       30,
-		InitialDelaySeconds: 5,
+		TimeoutSeconds: 5,
+		// [conductor]heartbeat_timeout is set to 120 so make PeriodSeconds
+		// more frequent to catch an offline conductor earlier
+		PeriodSeconds: 30,
 	}
-	readinessProbe := &corev1.Probe{
-		TimeoutSeconds:      5,
-		PeriodSeconds:       30,
-		InitialDelaySeconds: 5,
+	startupProbe := &corev1.Probe{
+		FailureThreshold: 30,
+		PeriodSeconds:    2,
 	}
 	dnsmasqLivenessProbe := &corev1.Probe{
 		TimeoutSeconds:      10,
@@ -89,24 +89,15 @@ func StatefulSet(
 	// https://kubernetes.io/docs/tasks/configure-pod-container/configure-liveness-readiness-startup-probes/
 	//
 
-	if instance.Spec.RPCTransport == "json-rpc" {
-		livenessProbe.TCPSocket = &corev1.TCPSocketAction{
-			Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(8089)},
-		}
-		readinessProbe.TCPSocket = &corev1.TCPSocketAction{
-			Port: intstr.IntOrString{Type: intstr.Int, IntVal: int32(8089)},
-		}
-	} else {
-		livenessProbe.Exec = &corev1.ExecAction{
-			Command: []string{
-				"/bin/true",
-			},
-		}
-		readinessProbe.Exec = &corev1.ExecAction{
-			Command: []string{
-				"/bin/true",
-			},
-		}
+	livenessProbe.Exec = &corev1.ExecAction{
+		Command: []string{
+			"/usr/local/bin/container-scripts/live_check",
+		},
+	}
+	startupProbe.Exec = &corev1.ExecAction{
+		Command: []string{
+			"/usr/local/bin/container-scripts/live_check",
+		},
 	}
 
 	httpbootLivenessProbe.TCPSocket = &corev1.TCPSocketAction{
@@ -188,12 +179,11 @@ func StatefulSet(
 		SecurityContext: &corev1.SecurityContext{
 			RunAsUser: &runAsUser,
 		},
-		Env:            env.MergeEnvs([]corev1.EnvVar{}, envVars),
-		VolumeMounts:   conductorVolumeMounts,
-		Resources:      instance.Spec.Resources,
-		ReadinessProbe: readinessProbe,
-		LivenessProbe:  livenessProbe,
-		// StartupProbe:   startupProbe,
+		Env:           env.MergeEnvs([]corev1.EnvVar{}, envVars),
+		VolumeMounts:  conductorVolumeMounts,
+		Resources:     instance.Spec.Resources,
+		LivenessProbe: livenessProbe,
+		StartupProbe:  startupProbe,
 	}
 	httpbootContainer := corev1.Container{
 		Name: "httpboot",

templates/ironicconductor/bin/live_check

@@ -0,0 +1,28 @@
+#!/bin/env python3
+#
+# Copyright 2025 Red Hat Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"); you may
+# not use this file except in compliance with the License. You may obtain
+# a copy of the License at
+#
+#      http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+# License for the specific language governing permissions and limitations
+# under the License.
+
+from ironic.common import config
+import ironic.conf
+CONF = ironic.conf.CONF
+
+config.parse_args([])
+from ironic.db.sqlalchemy import api
+
+dbapi = api.get_backend()
+if CONF.host in dbapi.get_online_conductors():
+    print(f'{CONF.host} is online')
+else:
+    raise Exception(f'{CONF.host} is offline')