fix: resolve webhook caBundle deadlock during helm upgrade

During helm upgrade, the webhook configurations' caBundle field was
reset to empty, causing new pods to fail readiness checks while the
old leader pod never detected the change (only watched the cert
Secret, with a 24h requeue). This created a deadlock where no pod
could fix the caBundle.

- Watch ValidatingWebhookConfiguration and MutatingWebhookConfiguration
  so the leader detects caBundle changes immediately
- Use bytes.Equal for caBundle comparison instead of len==0 so stale
  values are corrected, not just empty ones
- Remove caBundle from Helm webhook templates so upgrades stop
  resetting operator-managed values

Author: lockwobr

chart/templates/mutating-webhook.yaml

@@ -7,7 +7,6 @@ webhooks:
 - admissionReviewVersions:
   - v1
   clientConfig:
-    caBundle: ""
     service:
       name: skyhook-operator-webhook-service
       namespace: {{ .Release.Namespace }}
@@ -35,7 +34,6 @@ webhooks:
 - admissionReviewVersions:
   - v1
   clientConfig:
-    caBundle: ""
     service:
       name: skyhook-operator-webhook-service
       namespace: {{ .Release.Namespace }}

chart/templates/validating-webhook.yaml

@@ -7,7 +7,6 @@ webhooks:
 - admissionReviewVersions:
   - v1
   clientConfig:
-    caBundle: ""
     service:
       name: skyhook-operator-webhook-service
       namespace: {{ .Release.Namespace }}
@@ -34,7 +33,6 @@ webhooks:
 - admissionReviewVersions:
   - v1
   clientConfig:
-    caBundle: ""
     service:
       name: skyhook-operator-webhook-service
       namespace: {{ .Release.Namespace }}

operator/internal/controller/webhook_controller.go

@@ -27,6 +27,7 @@ import (
 	"time"
 
 	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 
 	"github.com/NVIDIA/skyhook/operator/api/v1alpha1"
@@ -127,9 +128,29 @@ func (r *WebhookController) SetupWithManager(mgr ctrl.Manager) error {
 		For(&corev1.Secret{}, builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {
 			return obj.GetNamespace() == r.namespace && obj.GetName() == r.opts.SecretName
 		}))).
+		// Watch webhook configurations so the leader detects external changes (e.g. Helm upgrade
+		// resetting caBundle) and fixes them immediately instead of waiting for the 24h requeue.
+		Watches(&admissionregistrationv1.ValidatingWebhookConfiguration{},
+			handler.EnqueueRequestsFromMapFunc(r.webhookConfigToSecret),
+			builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {
+				return obj.GetName() == validatingWebhookConfigName
+			}))).
+		Watches(&admissionregistrationv1.MutatingWebhookConfiguration{},
+			handler.EnqueueRequestsFromMapFunc(r.webhookConfigToSecret),
+			builder.WithPredicates(predicate.NewPredicateFuncs(func(obj client.Object) bool {
+				return obj.GetName() == mutatingWebhookConfigName
+			}))).
 		Complete(r)
 }
 
+// webhookConfigToSecret maps webhook config change events back to the cert Secret,
+// so the existing Reconcile() can verify and fix the caBundle.
+func (r *WebhookController) webhookConfigToSecret(_ context.Context, _ client.Object) []reconcile.Request {
+	return []reconcile.Request{{
+		NamespacedName: types.NamespacedName{Name: r.opts.SecretName, Namespace: r.namespace},
+	}}
+}
+
 // permissions
 //+kubebuilder:rbac:groups=admissionregistration.k8s.io,resources=validatingwebhookconfigurations;mutatingwebhookconfigurations,verbs=get;list;watch;create;update;patch;delete
 //+kubebuilder:rbac:groups=core,resources=secrets,verbs=get;list;watch;create;update;patch;delete
@@ -480,8 +501,8 @@ func deploymentPolicyMutatingRules() []admissionregistrationv1.RuleWithOperation
 func validatingWebhookNeedsUpdate(webhook *admissionregistrationv1.ValidatingWebhook, caBundle []byte, expectedRules []admissionregistrationv1.RuleWithOperations) bool {
 	needUpdate := false
 
-	// Check if CABundle needs to be set
-	if len(webhook.ClientConfig.CABundle) == 0 {
+	// Check if CABundle needs to be set or updated (catches both empty and stale values)
+	if !bytes.Equal(webhook.ClientConfig.CABundle, caBundle) {
 		webhook.ClientConfig.CABundle = caBundle
 		needUpdate = true
 	}
@@ -499,8 +520,8 @@ func validatingWebhookNeedsUpdate(webhook *admissionregistrationv1.ValidatingWeb
 func mutatingWebhookNeedsUpdate(webhook *admissionregistrationv1.MutatingWebhook, caBundle []byte, expectedRules []admissionregistrationv1.RuleWithOperations) bool {
 	needUpdate := false
 
-	// Check if CABundle needs to be set
-	if len(webhook.ClientConfig.CABundle) == 0 {
+	// Check if CABundle needs to be set or updated (catches both empty and stale values)
+	if !bytes.Equal(webhook.ClientConfig.CABundle, caBundle) {
 		webhook.ClientConfig.CABundle = caBundle
 		needUpdate = true
 	}

operator/internal/controller/webhook_controller_test.go

@@ -293,6 +293,32 @@ var _ = Describe("WebhookController", Ordered, func() {
 			Expect(needsUpdate).To(BeTrue(), "should detect empty CABundle")
 			Expect(webhook.ClientConfig.CABundle).To(Equal(caBundle), "CABundle should be updated")
 		})
+
+		It("should update CABundle when stale (non-empty but wrong)", func() {
+			expectedRules := skyhookRules()
+			correctCA := []byte("correct-ca")
+			staleCA := []byte("stale-ca-from-previous-cert")
+
+			validatingWebhook := admissionregistrationv1.ValidatingWebhook{
+				ClientConfig: admissionregistrationv1.WebhookClientConfig{
+					CABundle: staleCA,
+				},
+				Rules: expectedRules,
+			}
+			needsUpdate := validatingWebhookNeedsUpdate(&validatingWebhook, correctCA, expectedRules)
+			Expect(needsUpdate).To(BeTrue(), "should detect stale validating CABundle")
+			Expect(validatingWebhook.ClientConfig.CABundle).To(Equal(correctCA))
+
+			mutatingWebhook := admissionregistrationv1.MutatingWebhook{
+				ClientConfig: admissionregistrationv1.WebhookClientConfig{
+					CABundle: staleCA,
+				},
+				Rules: expectedRules,
+			}
+			needsUpdate = mutatingWebhookNeedsUpdate(&mutatingWebhook, correctCA, expectedRules)
+			Expect(needsUpdate).To(BeTrue(), "should detect stale mutating CABundle")
+			Expect(mutatingWebhook.ClientConfig.CABundle).To(Equal(correctCA))
+		})
 	})
 
 	Describe("Disk and Secret-to-Disk Sync Logic", func() {