Fix service-CA certificate rotation not triggering pod restarts

When service-CA rotates certificates, the cluster-monitoring-operator
wasn't watching service-CA generated secrets, causing pods to continue
using expired certificates until manually restarted.

This change adds detect changes happeing  for service-CA secrets in
monitoring namespaces (secrets ending with -tls or -cert) and triggers
reconciliation when they change, which triggers pod restarts and
certificate pickup.

Fixes the issue where EUS clusters running 3+ years without upgrades
require manual intervention after service-CA rotation.

Author: danielmellado

pkg/operator/operator.go

@@ -622,6 +622,16 @@ func (o *Operator) keyFunc(obj interface{}) (string, bool) {
 	return k, true
 }
 
+// isServiceCASecret checks if the given key represents a service-CA generated secret
+// that should trigger reconciliation when rotated.
+func (o *Operator) isServiceCASecret(key string) bool {
+	if strings.HasPrefix(key, o.namespace+"/") || strings.HasPrefix(key, o.namespaceUserWorkload+"/") {
+		secretName := strings.Split(key, "/")[1]
+		return strings.HasSuffix(secretName, "-tls") || strings.HasSuffix(secretName, "-cert")
+	}
+	return false
+}
+
 func (o *Operator) handleEvent(obj interface{}) {
 	cmoConfigMap := o.namespace + "/" + o.configMapName
 
@@ -671,6 +681,10 @@ func (o *Operator) handleEvent(obj interface{}) {
 	case federateClientCerts:
 	case uwmConfigMap:
 	default:
+		if o.isServiceCASecret(key) {
+			klog.V(4).Infof("Service-CA secret updated, triggering reconciliation: %s", key)
+			break
+		}
 		klog.V(5).Infof("ConfigMap or Secret (%s) not triggering an update.", key)
 		return
 	}

test/e2e/service_ca_rotation_test.go

@@ -0,0 +1,90 @@
+// Copyright 2021 The Cluster Monitoring Operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/openshift/cluster-monitoring-operator/test/e2e/framework"
+)
+
+func TestServiceCASecretRotation(t *testing.T) {
+	ctx := context.Background()
+
+	testCases := []struct {
+		name       string
+		secretName string
+		namespace  string
+	}{
+		{
+			name:       "monitoring-plugin-cert",
+			secretName: "monitoring-plugin-cert",
+			namespace:  f.Ns,
+		},
+		{
+			name:       "prometheus-k8s-tls",
+			secretName: "prometheus-k8s-tls",
+			namespace:  f.Ns,
+		},
+		{
+			name:       "alertmanager-main-tls",
+			secretName: "alertmanager-main-tls",
+			namespace:  f.Ns,
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			// Get the current secret
+			s, err := f.KubeClient.CoreV1().Secrets(tc.namespace).Get(ctx, tc.secretName, metav1.GetOptions{})
+			if err != nil {
+				t.Skipf("secret %s/%s not found, skipping test: %v", tc.namespace, tc.secretName, err)
+			}
+
+			if s.Annotations == nil {
+				s.Annotations = make(map[string]string)
+			}
+
+			s.Annotations["test.openshift.io/service-ca-test-rotation"] = "true"
+
+			if err := f.OperatorClient.CreateOrUpdateSecret(ctx, s); err != nil {
+				t.Fatalf("error updating secret %s/%s: %v", tc.namespace, tc.secretName, err)
+			}
+
+			// Wait for the cluster-monitoring-operator to detect the change and trigger reconciliation
+			err = framework.Poll(time.Second, 5*time.Minute, func() error {
+				s, err := f.KubeClient.CoreV1().Secrets(tc.namespace).Get(ctx, tc.secretName, metav1.GetOptions{})
+				if err != nil {
+					return fmt.Errorf("error loading secret %s/%s: %w", tc.namespace, tc.secretName, err)
+				}
+
+				if _, ok := s.Annotations["test.openshift.io/service-ca-test-rotation"]; ok {
+					return errors.New("rotation did not execute: service-ca-test-rotation annotation set")
+				}
+
+				return nil
+			})
+			if err != nil {
+				t.Fatal(err)
+			}
+		})
+	}
+}