Merge pull request noobaa#1684 from OdedViner/liveness_probe

add liveness, readiness, and startup probes to noobaa-operator

Author: dannyzaken

deploy/operator.yaml

@@ -42,6 +42,9 @@ spec:
         - name: noobaa-operator
           image: NOOBAA_OPERATOR_IMAGE
           terminationMessagePolicy: FallbackToLogsOnError
+          ports:
+            - name: readyz
+              containerPort: 8081
           volumeMounts:
           - name: bound-sa-token
             mountPath: /var/run/secrets/openshift/serviceaccount
@@ -51,6 +54,29 @@ spec:
           # SHOULD BE RETURNED ONCE COSI IS BACK
           # - name: socket
           #   mountPath: /var/lib/cosi
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: readyz
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          livenessProbe:
+            httpGet:
+              path: /readyz
+              port: readyz
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          startupProbe:
+            httpGet:
+              path: /readyz
+              port: readyz
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 30
           resources:
             limits:
               cpu: "250m"

pkg/bundle/deploy.go

@@ -6390,7 +6390,7 @@ spec:
   sourceNamespace: default
 `
 
-const Sha256_deploy_operator_yaml = "d78f7c32af03516bc70f0bddc65ad4837a138c9bd334676a020d870edf9a8467"
+const Sha256_deploy_operator_yaml = "44022d49f1e87196410ff28589fca9001c25928957dfa3db208eab58220d364d"
 
 const File_deploy_operator_yaml = `apiVersion: apps/v1
 kind: Deployment
@@ -6436,6 +6436,9 @@ spec:
         - name: noobaa-operator
           image: NOOBAA_OPERATOR_IMAGE
           terminationMessagePolicy: FallbackToLogsOnError
+          ports:
+            - name: readyz
+              containerPort: 8081
           volumeMounts:
           - name: bound-sa-token
             mountPath: /var/run/secrets/openshift/serviceaccount
@@ -6445,6 +6448,29 @@ spec:
           # SHOULD BE RETURNED ONCE COSI IS BACK
           # - name: socket
           #   mountPath: /var/lib/cosi
+          readinessProbe:
+            httpGet:
+              path: /readyz
+              port: readyz
+            initialDelaySeconds: 5
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          livenessProbe:
+            httpGet:
+              path: /readyz
+              port: readyz
+            initialDelaySeconds: 15
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 3
+          startupProbe:
+            httpGet:
+              path: /readyz
+              port: readyz
+            periodSeconds: 10
+            timeoutSeconds: 5
+            failureThreshold: 30
           resources:
             limits:
               cpu: "250m"

pkg/operator/manager.go

@@ -24,6 +24,7 @@ import (
 
 	"github.com/operator-framework/operator-lib/leader"
 	"sigs.k8s.io/controller-runtime/pkg/cache"
+	"sigs.k8s.io/controller-runtime/pkg/healthz"
 	"sigs.k8s.io/controller-runtime/pkg/manager"
 	"sigs.k8s.io/controller-runtime/pkg/manager/signals"
 	metricsServer "sigs.k8s.io/controller-runtime/pkg/metrics/server"
@@ -44,6 +45,11 @@ func RunOperator(cmd *cobra.Command, args []string) {
 		util.InitLogger(logrus.DebugLevel)
 	}
 	version.RunVersion(cmd, args)
+	// Probe address from CLI flag (defaults to :8081)
+	probeAddr := os.Getenv("HEALTH_PROBE_BIND_ADDRESS")
+	if probeAddr == "" {
+		probeAddr = ":8081"
+	}
 
 	config := util.KubeConfig()
 
@@ -66,6 +72,7 @@ func RunOperator(cmd *cobra.Command, args []string) {
 		Metrics: metricsServer.Options{
 			BindAddress: fmt.Sprintf("%s:%d", metricsHost, metricsPort),
 		},
+		HealthProbeBindAddress: probeAddr, // Serve /healthz and /readyz here
 	})
 	if err != nil {
 		log.Fatalf("Failed to create manager: %s", err)
@@ -97,6 +104,11 @@ func RunOperator(cmd *cobra.Command, args []string) {
 		log.Fatalf("Failed AddToClusterScopedManager: %s", err)
 	}
 
+	// Register readiness endpoint on mgr
+	if err := mgr.AddReadyzCheck("readyz", healthz.Ping); err != nil {
+		log.Fatalf("Failed to add readiness check: %s", err)
+	}
+
 	util.Panic(mgr.Add(manager.RunnableFunc(func(ctx context.Context) error {
 		system.RunOperatorCreate(cmd, args)
 		return nil