OCPBUGS-58145: reload serving cert on rotation

pkg/operator/controller/canary/controller.go

@@ -35,6 +35,8 @@ import (
 	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/controller-runtime/pkg/source"
+
+	"k8s.io/client-go/tools/record"
 )
 
 const (
@@ -80,6 +82,7 @@ func New(mgr manager.Manager, config Config) (controller.Controller, error) {
 	reconciler := &reconciler{
 		config:                    config,
 		client:                    mgr.GetClient(),
+		recorder:                  mgr.GetEventRecorderFor(canaryControllerName),
 		enableCanaryRouteRotation: false,
 	}
 	c, err := controller.New(canaryControllerName, mgr, controller.Options{Reconciler: reconciler})
@@ -161,6 +164,15 @@ func New(mgr manager.Manager, config Config) (controller.Controller, error) {
 		return nil, err
 	}
 
+	// Watch the canary serving cert secret and enqueue the default ingress controller so
+	// that changes to the serving cert cause the canary daemonset to be reconciled.
+	canarySecretPredicate := predicate.NewPredicateFuncs(func(o client.Object) bool {
+		return o.GetNamespace() == operatorcontroller.DefaultCanaryNamespace && o.GetName() == "canary-serving-cert"
+	})
+	if err := c.Watch(source.Kind[client.Object](operatorCache, &corev1.Secret{}, enqueueRequestForDefaultIngressController(config.Namespace), canarySecretPredicate)); err != nil {
+		return nil, err
+	}
+
 	return c, nil
 }
 
@@ -252,7 +264,8 @@ type Config struct {
 type reconciler struct {
 	config Config
 
-	client client.Client
+	client   client.Client
+	recorder record.EventRecorder
 
 	// Use a mutex so enableCanaryRotation is
 	// go-routine safe.

pkg/operator/controller/canary/daemonset.go

@@ -13,11 +13,31 @@ import (
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/types"
 )
 
 // ensureCanaryDaemonSet ensures the canary daemonset exists
 func (r *reconciler) ensureCanaryDaemonSet() (bool, *appsv1.DaemonSet, error) {
-	desired := desiredCanaryDaemonSet(r.config.CanaryImage)
+	// Attempt to read the canary serving cert secret and compute a content hash.
+	// If the secret is missing or incomplete, proceed without the annotation but
+	// surface a log entry so operators can investigate.
+	var certHash string
+	secret := &corev1.Secret{}
+	if err := r.client.Get(context.TODO(), types.NamespacedName{Namespace: controller.DefaultCanaryNamespace, Name: "canary-serving-cert"}, secret); err != nil {
+		if errors.IsNotFound(err) {
+			log.Info("canary serving cert secret not found; skipping canary-serving-cert-hash annotation")
+		} else {
+			return false, nil, fmt.Errorf("failed to get canary serving cert secret: %v", err)
+		}
+	} else {
+		if h, err := ComputeTLSSecretHash(secret); err != nil {
+			log.Info("canary serving cert secret is incomplete; skipping canary-serving-cert-hash annotation", "error", err)
+		} else {
+			certHash = h
+		}
+	}
+
+	desired := desiredCanaryDaemonSet(r.config.CanaryImage, certHash)
 	haveDs, current, err := r.currentCanaryDaemonSet()
 	if err != nil {
 		return false, nil, err
@@ -70,17 +90,40 @@ func (r *reconciler) updateCanaryDaemonSet(current, desired *appsv1.DaemonSet) (
 		return false, nil
 	}
 
+	// Capture annotation change for events.
+	var oldHash, newHash string
+	if current.Spec.Template.Annotations != nil {
+		oldHash = current.Spec.Template.Annotations[CanaryServingCertHashAnnotation]
+	}
+	if updated.Spec.Template.Annotations != nil {
+		newHash = updated.Spec.Template.Annotations[CanaryServingCertHashAnnotation]
+	}
+
 	diff := cmp.Diff(current, updated, cmpopts.EquateEmpty())
 	if err := r.client.Update(context.TODO(), updated); err != nil {
 		return false, fmt.Errorf("failed to update canary daemonset %s/%s: %v", updated.Namespace, updated.Name, err)
 	}
+
 	log.Info("updated canary daemonset", "namespace", updated.Namespace, "name", updated.Name, "diff", diff)
+
+	// If the only meaningful change (or one of the changes) was the canary cert
+	// annotation, emit an event for traceability.
+	if newHash != "" && newHash != oldHash {
+		short := newHash
+		if len(short) > 8 {
+			short = short[:8]
+		}
+		if r.recorder != nil {
+			r.recorder.Eventf(updated, "Normal", "CanaryCertRotated", "Canary serving cert rotated, updated pod template annotation hash: %s", short)
+		}
+	}
+
 	return true, nil
 }
 
 // desiredCanaryDaemonSet returns the desired canary daemonset read in
 // from manifests
-func desiredCanaryDaemonSet(canaryImage string) *appsv1.DaemonSet {
+func desiredCanaryDaemonSet(canaryImage string, certHash string) *appsv1.DaemonSet {
 	daemonset := manifests.CanaryDaemonSet()
 	name := controller.CanaryDaemonSetName()
 	daemonset.Name = name.Name
@@ -97,6 +140,13 @@ func desiredCanaryDaemonSet(canaryImage string) *appsv1.DaemonSet {
 	daemonset.Spec.Template.Spec.Containers[0].Image = canaryImage
 	daemonset.Spec.Template.Spec.Containers[0].Command = []string{"ingress-operator", CanaryHealthcheckCommand}
 
+	if certHash != "" {
+		if daemonset.Spec.Template.Annotations == nil {
+			daemonset.Spec.Template.Annotations = map[string]string{}
+		}
+		daemonset.Spec.Template.Annotations[CanaryServingCertHashAnnotation] = certHash
+	}
+
 	return daemonset
 }
 
@@ -163,6 +213,26 @@ func canaryDaemonSetChanged(current, expected *appsv1.DaemonSet) (bool, *appsv1.
 		changed = true
 	}
 
+	// Update when the canary-serving-cert hash annotation changes on the pod template.
+	var currentHash, expectedHash string
+	if current.Spec.Template.Annotations != nil {
+		currentHash = current.Spec.Template.Annotations[CanaryServingCertHashAnnotation]
+	}
+	if expected.Spec.Template.Annotations != nil {
+		expectedHash = expected.Spec.Template.Annotations[CanaryServingCertHashAnnotation]
+	}
+	if currentHash != expectedHash {
+		if updated.Spec.Template.Annotations == nil {
+			updated.Spec.Template.Annotations = map[string]string{}
+		}
+		if expectedHash == "" {
+			delete(updated.Spec.Template.Annotations, CanaryServingCertHashAnnotation)
+		} else {
+			updated.Spec.Template.Annotations[CanaryServingCertHashAnnotation] = expectedHash
+		}
+		changed = true
+	}
+
 	if !changed {
 		return false, nil
 	}

pkg/operator/controller/canary/daemonset_test.go

@@ -16,7 +16,7 @@ import (
 func Test_desiredCanaryDaemonSet(t *testing.T) {
 	// canaryImageName is the ingress-operator image
 	canaryImageName := "openshift/origin-cluster-ingress-operator:latest"
-	daemonset := desiredCanaryDaemonSet(canaryImageName)
+	daemonset := desiredCanaryDaemonSet(canaryImageName, "")
 
 	expectedDaemonSetName := controller.CanaryDaemonSetName()
 
@@ -225,11 +225,21 @@ func Test_canaryDaemonsetChanged(t *testing.T) {
 			},
 			expect: true,
 		},
+		{
+			description: "if canary serving cert annotation changes",
+			mutate: func(ds *appsv1.DaemonSet) {
+				if ds.Spec.Template.Annotations == nil {
+					ds.Spec.Template.Annotations = map[string]string{}
+				}
+				ds.Spec.Template.Annotations[CanaryServingCertHashAnnotation] = "d34db33f"
+			},
+			expect: true,
+		},
 	}
 
 	for _, tc := range testCases {
 		t.Run(tc.description, func(t *testing.T) {
-			original := desiredCanaryDaemonSet("")
+			original := desiredCanaryDaemonSet("", "")
 			mutated := original.DeepCopy()
 			tc.mutate(mutated)
 			if changed, updated := canaryDaemonSetChanged(original, mutated); changed != tc.expect {

pkg/operator/controller/canary/secret_hash.go

@@ -0,0 +1,47 @@
+package canary
+
+import (
+	"crypto/sha256"
+	"encoding/hex"
+	"fmt"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+const (
+	// CanaryServingCertHashAnnotation is the annotation key used on the
+	// canary DaemonSet PodTemplate to force a rollout when the canary
+	// serving cert secret changes.
+	CanaryServingCertHashAnnotation = "ingress.operator.openshift.io/canary-serving-cert-hash"
+)
+
+// ComputeTLSSecretHash computes a stable sha256 hex string over the
+// relevant keys in a TLS secret. Required keys are `tls.crt` and
+// `tls.key`. `ca.crt` is included if present.
+func ComputeTLSSecretHash(secret *corev1.Secret) (string, error) {
+	if secret == nil {
+		return "", fmt.Errorf("secret is nil")
+	}
+
+	if secret.Data == nil {
+		return "", fmt.Errorf("secret has no data")
+	}
+
+	crt, okCrt := secret.Data["tls.crt"]
+	key, okKey := secret.Data["tls.key"]
+	if !okCrt || !okKey {
+		return "", fmt.Errorf("secret missing tls.crt or tls.key")
+	}
+
+	hasher := sha256.New()
+	hasher.Write(crt)
+	hasher.Write([]byte("|"))
+	hasher.Write(key)
+	if ca, ok := secret.Data["ca.crt"]; ok {
+		hasher.Write([]byte("|"))
+		hasher.Write(ca)
+	}
+
+	sum := hasher.Sum(nil)
+	return hex.EncodeToString(sum), nil
+}

pkg/operator/controller/canary/secret_hash_test.go

@@ -0,0 +1,66 @@
+package canary
+
+import (
+	"testing"
+
+	corev1 "k8s.io/api/core/v1"
+)
+
+func Test_ComputeTLSSecretHash(t *testing.T) {
+	base := &corev1.Secret{
+		Data: map[string][]byte{
+			"tls.crt": []byte("cert-data"),
+			"tls.key": []byte("key-data"),
+		},
+	}
+
+	same := &corev1.Secret{
+		Data: map[string][]byte{
+			"tls.crt": []byte("cert-data"),
+			"tls.key": []byte("key-data"),
+		},
+	}
+
+	different := &corev1.Secret{
+		Data: map[string][]byte{
+			"tls.crt": []byte("other-cert"),
+			"tls.key": []byte("key-data"),
+		},
+	}
+
+	withCA := &corev1.Secret{
+		Data: map[string][]byte{
+			"tls.crt": []byte("cert-data"),
+			"tls.key": []byte("key-data"),
+			"ca.crt":  []byte("ca-data"),
+		},
+	}
+
+	h1, err := ComputeTLSSecretHash(base)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	h2, err := ComputeTLSSecretHash(same)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if h1 != h2 {
+		t.Fatalf("expected same hash for identical secrets: %s != %s", h1, h2)
+	}
+
+	hd, err := ComputeTLSSecretHash(different)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if h1 == hd {
+		t.Fatalf("expected different hash for different secret data")
+	}
+
+	hca, err := ComputeTLSSecretHash(withCA)
+	if err != nil {
+		t.Fatalf("unexpected error: %v", err)
+	}
+	if h1 == hca {
+		t.Fatalf("expected different hash when ca.crt is present")
+	}
+}