Recover from panic

internal/controller/atom_controller.go

@@ -30,6 +30,8 @@ import (
 	"fmt"
 	"time"
 
+	"github.com/pkg/errors"
+
 	policyv1 "k8s.io/api/policy/v1"
 	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
@@ -132,6 +134,14 @@ func (r *AtomReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul
 		return result, client.IgnoreNotFound(err)
 	}
 
+	// Recover from a panic so we can add the error to the status of the Atom
+	defer func() {
+		if rec := recover(); rec != nil {
+			err = recoveredPanicToError(rec)
+			r.logAndUpdateStatusError(ctx, atom, err)
+		}
+	}()
+
 	// Check TTL expiry
 	if ttlExpired(atom) {
 		err = r.Client.Delete(ctx, atom)
@@ -150,7 +160,6 @@ func (r *AtomReconciler) Reconcile(ctx context.Context, req ctrl.Request) (resul
 	r.logAndUpdateStatusFinished(ctx, atom, operationResults)
 
 	return result, err
-
 }
 
 func (r *AtomReconciler) createOrUpdateAllForAtom(ctx context.Context, atom *pdoknlv3.Atom, ownerInfo *smoothoperatorv1.OwnerInfo) (operationResults map[string]controllerutil.OperationResult, err error) {
@@ -276,3 +285,20 @@ func ttlExpired(atom *pdoknlv3.Atom) bool {
 
 	return false
 }
+
+func recoveredPanicToError(rec any) (err error) {
+	switch x := rec.(type) {
+	case string:
+		err = errors.New(x)
+	case error:
+		err = x
+	default:
+		err = errors.New("unknown panic")
+	}
+
+	// Add stack
+	// TODO - this doesn't seem to work, see if there is a better method to add the stack
+	err = errors.WithStack(err)
+
+	return
+}

internal/controller/status.go

@@ -15,6 +15,9 @@ import (
 )
 
 func (r *AtomReconciler) logAndUpdateStatusError(ctx context.Context, atom *pdoknlv3.Atom, err error) {
+	lgr := logf.FromContext(ctx)
+	lgr.Error(err, "reconcile error")
+
 	r.updateStatus(ctx, atom, []metav1.Condition{{
 		Type:               reconciledConditionType,
 		Status:             metav1.ConditionFalse,