Don't repanic in the defer() function

A panic would crash the controller manager binary which means all
controllers in the controller manager. We should rather handle it
gracefully.

This is a pattern used in all openstack operators and should probably
be changed in all of them.

Assisted-by: Claude-4-sonnet
Signed-off-by: rabi <ramishra@redhat.com>

Author: rabi

controllers/openstackbaremetalset_controller.go

@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"reflect"
 	"time"
@@ -50,6 +51,11 @@ import (
 	"github.com/openstack-k8s-operators/openstack-baremetal-operator/pkg/openstackbaremetalset"
 )
 
+var (
+	// ErrBaremetalSetReconciliationPanic indicates a panic occurred during BaremetalSet reconciliation
+	ErrBaremetalSetReconciliationPanic = errors.New("baremetal set reconciliation panic occurred")
+)
+
 // OpenStackBaremetalSetReconciler reconciles a OpenStackBaremetalSet object
 type OpenStackBaremetalSetReconciler struct {
 	client.Client
@@ -112,17 +118,28 @@ func (r *OpenStackBaremetalSetReconciler) Reconcile(ctx context.Context, req ctr
 	// Always patch the instance status when exiting this function so we can
 	// persist any changes.
 	defer func() {
-		// Don't update the status, if reconciler Panics
+		// Handle reconciler panics gracefully to prevent controller crash
 		if rc := recover(); rc != nil {
-			r.Log.Info(fmt.Sprintf("panic during reconcile %v\n", rc))
-			panic(rc)
+			// Log detailed panic information
+			r.Log.Error(ErrBaremetalSetReconciliationPanic,
+				"Panic occurred during OpenStackBaremetalSet reconciliation",
+				"instance", instance.Name,
+				"namespace", instance.Namespace,
+				"panic", rc)
+
+			// Set return error to trigger requeue with backoff
+			_err = fmt.Errorf("%w: %v", ErrBaremetalSetReconciliationPanic, rc)
+			return // Skip status update - reconciliation incomplete, let controller-runtime handle requeue
 		}
+
 		condition.RestoreLastTransitionTimes(
 			&instance.Status.Conditions, savedConditions)
 		if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) {
 			instance.Status.Conditions.Set(
 				instance.Status.Conditions.Mirror(condition.ReadyCondition))
 		}
+
+		// Update status only if no panic occurred
 		err := helper.PatchInstance(ctx, instance)
 		if err != nil {
 			_err = err

controllers/openstackprovisionserver_controller.go

@@ -18,6 +18,7 @@ package controllers
 
 import (
 	"context"
+	"errors"
 	"fmt"
 	"net"
 	"strconv"
@@ -61,6 +62,9 @@ var (
 		Version:  "v1alpha1",
 		Resource: "provisionings",
 	}
+
+	// ErrProvisionServerReconciliationPanic indicates a panic occurred during ProvisionServer reconciliation
+	ErrProvisionServerReconciliationPanic = errors.New("provision server reconciliation panic occurred")
 )
 
 // OpenStackProvisionServerReconciler reconciles a OpenStackProvisionServer object
@@ -135,17 +139,28 @@ func (r *OpenStackProvisionServerReconciler) Reconcile(ctx context.Context, req
 	// Always patch the instance status when exiting this function so we can
 	// persist any changes.
 	defer func() {
-		// Don't update the status, if reconciler Panics
+		// Handle reconciler panics gracefully to prevent controller crash
 		if rc := recover(); rc != nil {
-			r.Log.Info(fmt.Sprintf("panic during reconcile %v\n", rc))
-			panic(rc)
+			// Log detailed panic information
+			r.Log.Error(ErrProvisionServerReconciliationPanic,
+				"Panic occurred during OpenStackProvisionServer reconciliation",
+				"instance", instance.Name,
+				"namespace", instance.Namespace,
+				"panic", rc)
+
+			// Set return error to trigger requeue with backoff
+			_err = fmt.Errorf("%w: %v", ErrProvisionServerReconciliationPanic, rc)
+			return // Skip status update - reconciliation incomplete, let controller-runtime handle requeue
 		}
+
 		condition.RestoreLastTransitionTimes(
 			&instance.Status.Conditions, savedConditions)
 		if instance.Status.Conditions.IsUnknown(condition.ReadyCondition) {
 			instance.Status.Conditions.Set(
 				instance.Status.Conditions.Mirror(condition.ReadyCondition))
 		}
+
+		// Update status only if no panic occurred
 		err := helper.PatchInstance(ctx, instance)
 		if err != nil {
 			_err = err