Fix service-CA certificate rotation not triggering pod restarts

When service-CA rotates certificates, the cluster-monitoring-operator
wasn't watching service-CA generated secrets, causing pods to continue
using expired certificates until manually restarted.

This change adds detect changes happeing  for service-CA secrets in
monitoring namespaces (secrets ending with -tls or -cert) and triggers
reconciliation when they change, which triggers pod restarts and
certificate pickup.

Fixes the issue where EUS clusters running 3+ years without upgrades
require manual intervention after service-CA rotation.

Signed-off-by: Daniel Mellado <[email protected]>

Author: danielmellado

pkg/operator/operator.go

@@ -622,6 +622,16 @@ func (o *Operator) keyFunc(obj interface{}) (string, bool) {
 	return k, true
 }
 
+// isServiceCASecret checks if the given key represents a service-CA generated secret
+// that should trigger reconciliation when rotated.
+func (o *Operator) isServiceCASecret(key string) bool {
+	if strings.HasPrefix(key, o.namespace+"/") || strings.HasPrefix(key, o.namespaceUserWorkload+"/") {
+		secretName := strings.Split(key, "/")[1]
+		return strings.HasSuffix(secretName, "-tls") || strings.HasSuffix(secretName, "-cert")
+	}
+	return false
+}
+
 func (o *Operator) handleEvent(obj interface{}) {
 	cmoConfigMap := o.namespace + "/" + o.configMapName
 
@@ -671,6 +681,10 @@ func (o *Operator) handleEvent(obj interface{}) {
 	case federateClientCerts:
 	case uwmConfigMap:
 	default:
+		if o.isServiceCASecret(key) {
+			klog.V(4).Infof("Service-CA secret updated, triggering reconciliation: %s", key)
+			break
+		}
 		klog.V(5).Infof("ConfigMap or Secret (%s) not triggering an update.", key)
 		return
 	}

test/e2e/service_ca_rotation_test.go

@@ -0,0 +1,103 @@
+// Copyright 2021 The Cluster Monitoring Operator Authors
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+
+	"github.com/openshift/cluster-monitoring-operator/test/e2e/framework"
+)
+
+func TestServiceCASecretRotation(t *testing.T) {
+	ctx := context.Background()
+
+	testCases := []struct {
+		name       string
+		secretName string
+	}{
+		{
+			name:       "monitoring-plugin-cert change is handled gracefully",
+			secretName: "monitoring-plugin-cert",
+		},
+	}
+
+	for _, tc := range testCases {
+		t.Run(tc.name, func(t *testing.T) {
+			s, err := f.KubeClient.CoreV1().Secrets(f.Ns).Get(ctx, tc.secretName, metav1.GetOptions{})
+			if err != nil {
+				t.Skipf("secret %s/%s not found, skipping test: %v", f.Ns, tc.secretName, err)
+			}
+
+			t.Cleanup(func() {
+				if s, err := f.KubeClient.CoreV1().Secrets(f.Ns).Get(ctx, tc.secretName, metav1.GetOptions{}); err == nil && s.Annotations != nil {
+					if _, exists := s.Annotations["test.openshift.io/service-ca-test-rotation"]; exists {
+						delete(s.Annotations, "test.openshift.io/service-ca-test-rotation")
+						_ = f.OperatorClient.CreateOrUpdateSecret(ctx, s)
+					}
+				}
+			})
+
+			initialCO, err := f.OpenShiftConfigClient.ConfigV1().ClusterOperators().Get(ctx, "monitoring", metav1.GetOptions{})
+			if err != nil {
+				t.Fatalf("error getting initial cluster operator status: %v", err)
+			}
+
+			if s.Annotations == nil {
+				s.Annotations = make(map[string]string)
+			}
+			s.Annotations["test.openshift.io/service-ca-test-rotation"] = fmt.Sprintf("%d", time.Now().Unix())
+
+			if err := f.OperatorClient.CreateOrUpdateSecret(ctx, s); err != nil {
+				t.Fatalf("error updating secret %s/%s: %v", f.Ns, tc.secretName, err)
+			}
+
+			err = framework.Poll(5*time.Second, 2*time.Minute, func() error {
+				co, err := f.OpenShiftConfigClient.ConfigV1().ClusterOperators().Get(ctx, "monitoring", metav1.GetOptions{})
+				if err != nil {
+					return fmt.Errorf("error getting cluster operator status: %w", err)
+				}
+
+				for _, condition := range co.Status.Conditions {
+					switch condition.Type {
+					case "Available":
+						if condition.Status != "True" {
+							return fmt.Errorf("cluster operator is not available: %s", condition.Message)
+						}
+					case "Degraded":
+						if condition.Status == "True" {
+							return fmt.Errorf("cluster operator is degraded: %s", condition.Message)
+						}
+					case "Progressing":
+						if condition.Status == "True" && condition.Reason != "" {
+							if co.Generation == initialCO.Generation {
+								return fmt.Errorf("operator is progressing but generation hasn't changed")
+							}
+						}
+					}
+				}
+
+				return nil
+			})
+			if err != nil {
+				t.Fatalf("service-CA secret rotation test failed: %v", err)
+			}
+		})
+	}
+}