diff --git a/Documentation/deps-versions.md b/Documentation/deps-versions.md index 9820e36c70..11b119adf0 100644 --- a/Documentation/deps-versions.md +++ b/Documentation/deps-versions.md @@ -1,7 +1,7 @@ | OCP Version | alertmanager | kubeRbacProxy | kubeStateMetrics | kubernetesMetricsServer | monitoringPlugin | nodeExporter | promLabelProxy | prometheus | prometheusOperator | thanos | |--------------|----------------------------------------------------------------------------------|--------------------------------------------------------------------------|-----------------------------------------------------------------------------|-----------------------------------------------------------------------------------|---------------------------------------------------------------------------|-----------------------------------------------------------------------|---------------------------------------------------------------------------|---------------------------------------------------------------------|------------------------------------------------------------------------------|-----------------------------------------------------------------| -| release-4.21 | [0.28.1](https://github.com/openshift/prometheus-alertmanager/blob/release-4.21) | [0.19.1](https://github.com/openshift/kube-rbac-proxy/blob/release-4.21) | [2.16.0](https://github.com/openshift/kube-state-metrics/blob/release-4.21) | [0.8.0](https://github.com/openshift/kubernetes-metrics-server/blob/release-4.21) | [1.0.0](https://github.com/openshift/monitoring-plugin/blob/release-4.21) | [1.9.1](https://github.com/openshift/node_exporter/blob/release-4.21) | [0.11.1](https://github.com/openshift/prom-label-proxy/blob/release-4.21) | [3.3.1](https://github.com/openshift/prometheus/blob/release-4.21) | [0.81.0](https://github.com/openshift/prometheus-operator/blob/release-4.21) | [0.37.2](https://github.com/openshift/thanos/blob/release-4.21) | -| release-4.20 | [0.28.1](https://github.com/openshift/prometheus-alertmanager/blob/release-4.20) | [0.19.1](https://github.com/openshift/kube-rbac-proxy/blob/release-4.20) | [2.16.0](https://github.com/openshift/kube-state-metrics/blob/release-4.20) | [0.8.0](https://github.com/openshift/kubernetes-metrics-server/blob/release-4.20) | [1.0.0](https://github.com/openshift/monitoring-plugin/blob/release-4.20) | [1.9.1](https://github.com/openshift/node_exporter/blob/release-4.20) | [0.11.1](https://github.com/openshift/prom-label-proxy/blob/release-4.20) | [3.3.1](https://github.com/openshift/prometheus/blob/release-4.20) | [0.81.0](https://github.com/openshift/prometheus-operator/blob/release-4.20) | [0.37.2](https://github.com/openshift/thanos/blob/release-4.20) | +| release-4.21 | [0.28.1](https://github.com/openshift/prometheus-alertmanager/blob/release-4.21) | [0.19.1](https://github.com/openshift/kube-rbac-proxy/blob/release-4.21) | [2.16.0](https://github.com/openshift/kube-state-metrics/blob/release-4.21) | [0.8.0](https://github.com/openshift/kubernetes-metrics-server/blob/release-4.21) | [1.0.0](https://github.com/openshift/monitoring-plugin/blob/release-4.21) | [1.9.1](https://github.com/openshift/node_exporter/blob/release-4.21) | [0.11.1](https://github.com/openshift/prom-label-proxy/blob/release-4.21) | [3.3.1](https://github.com/openshift/prometheus/blob/release-4.21) | [0.84.0](https://github.com/openshift/prometheus-operator/blob/release-4.21) | [0.37.2](https://github.com/openshift/thanos/blob/release-4.21) | +| release-4.20 | [0.28.1](https://github.com/openshift/prometheus-alertmanager/blob/release-4.20) | [0.19.1](https://github.com/openshift/kube-rbac-proxy/blob/release-4.20) | [2.16.0](https://github.com/openshift/kube-state-metrics/blob/release-4.20) | [0.8.0](https://github.com/openshift/kubernetes-metrics-server/blob/release-4.20) | [1.0.0](https://github.com/openshift/monitoring-plugin/blob/release-4.20) | [1.9.1](https://github.com/openshift/node_exporter/blob/release-4.20) | [0.11.1](https://github.com/openshift/prom-label-proxy/blob/release-4.20) | [3.3.1](https://github.com/openshift/prometheus/blob/release-4.20) | [0.84.0](https://github.com/openshift/prometheus-operator/blob/release-4.20) | [0.37.2](https://github.com/openshift/thanos/blob/release-4.20) | | release-4.19 | [0.28.1](https://github.com/openshift/prometheus-alertmanager/blob/release-4.19) | [0.19.0](https://github.com/openshift/kube-rbac-proxy/blob/release-4.19) | [2.15.0](https://github.com/openshift/kube-state-metrics/blob/release-4.19) | [0.7.2](https://github.com/openshift/kubernetes-metrics-server/blob/release-4.19) | [1.0.0](https://github.com/openshift/monitoring-plugin/blob/release-4.19) | [1.9.1](https://github.com/openshift/node_exporter/blob/release-4.19) | [0.11.0](https://github.com/openshift/prom-label-proxy/blob/release-4.19) | [3.2.1](https://github.com/openshift/prometheus/blob/release-4.19) | [0.81.0](https://github.com/openshift/prometheus-operator/blob/release-4.19) | [0.37.2](https://github.com/openshift/thanos/blob/release-4.19) | | release-4.18 | [0.27.0](https://github.com/openshift/prometheus-alertmanager/blob/release-4.18) | [0.18.1](https://github.com/openshift/kube-rbac-proxy/blob/release-4.18) | [2.13.0](https://github.com/openshift/kube-state-metrics/blob/release-4.18) | [0.7.2](https://github.com/openshift/kubernetes-metrics-server/blob/release-4.18) | [1.0.0](https://github.com/openshift/monitoring-plugin/blob/release-4.18) | [1.8.2](https://github.com/openshift/node_exporter/blob/release-4.18) | [0.11.0](https://github.com/openshift/prom-label-proxy/blob/release-4.18) | [2.55.1](https://github.com/openshift/prometheus/blob/release-4.18) | [0.78.2](https://github.com/openshift/prometheus-operator/blob/release-4.18) | [0.36.1](https://github.com/openshift/thanos/blob/release-4.18) | | release-4.17 | [0.27.0](https://github.com/openshift/prometheus-alertmanager/blob/release-4.17) | [0.17.1](https://github.com/openshift/kube-rbac-proxy/blob/release-4.17) | [2.13.0](https://github.com/openshift/kube-state-metrics/blob/release-4.17) | [0.7.1](https://github.com/openshift/kubernetes-metrics-server/blob/release-4.17) | [1.0.0](https://github.com/openshift/monitoring-plugin/blob/release-4.17) | [1.8.2](https://github.com/openshift/node_exporter/blob/release-4.17) | [0.11.0](https://github.com/openshift/prom-label-proxy/blob/release-4.17) | [2.53.1](https://github.com/openshift/prometheus/blob/release-4.17) | [0.75.2](https://github.com/openshift/prometheus-operator/blob/release-4.17) | [0.35.1](https://github.com/openshift/thanos/blob/release-4.17) | diff --git a/jsonnet/versions.yaml b/jsonnet/versions.yaml index d2a07c4293..46b2b6e121 100644 --- a/jsonnet/versions.yaml +++ b/jsonnet/versions.yaml @@ -23,5 +23,5 @@ versions: promLabelProxy: 0.11.1 prometheus: 3.3.1 prometheusAdapter: 0.12.0 - prometheusOperator: 0.81.0 + prometheusOperator: 0.84.0 thanos: 0.37.2 diff --git a/pkg/tasks/alertmanager.go b/pkg/tasks/alertmanager.go index 39eedb2e7d..22757b1701 100644 --- a/pkg/tasks/alertmanager.go +++ b/pkg/tasks/alertmanager.go @@ -101,7 +101,7 @@ func (t *AlertmanagerTask) create(ctx context.Context) error { return fmt.Errorf("initializing Alertmanager RBAC proxy Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, rs) + err = t.client.CreateOrUpdateSecret(ctx, rs) if err != nil { return fmt.Errorf("creating Alertmanager RBAC proxy Secret failed: %w", err) } @@ -111,7 +111,7 @@ func (t *AlertmanagerTask) create(ctx context.Context) error { return fmt.Errorf("initializing Alertmanager RBAC proxy metric Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, rsm) + err = t.client.CreateOrUpdateSecret(ctx, rsm) if err != nil { return fmt.Errorf("creating Alertmanager RBAC proxy metric Secret failed: %w", err) } @@ -163,7 +163,7 @@ func (t *AlertmanagerTask) create(ctx context.Context) error { return fmt.Errorf("initializing Alertmanager proxy web Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, ps) + err = t.client.CreateOrUpdateSecret(ctx, ps) if err != nil { return fmt.Errorf("creating Alertmanager proxy web Secret failed: %w", err) } diff --git a/pkg/tasks/alertmanager_user_workload.go b/pkg/tasks/alertmanager_user_workload.go index c5cb88a86b..05955e285e 100644 --- a/pkg/tasks/alertmanager_user_workload.go +++ b/pkg/tasks/alertmanager_user_workload.go @@ -80,7 +80,7 @@ func (t *AlertmanagerUserWorkloadTask) create(ctx context.Context) error { return fmt.Errorf("initializing Alertmanager User Workload RBAC proxy Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, s) + err = t.client.CreateOrUpdateSecret(ctx, s) if err != nil { return fmt.Errorf("creating Alertmanager User Workload RBAC proxy Secret failed: %w", err) } @@ -90,7 +90,7 @@ func (t *AlertmanagerUserWorkloadTask) create(ctx context.Context) error { return fmt.Errorf("initializing Alertmanager User Workload RBAC proxy tenancy Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, s) + err = t.client.CreateOrUpdateSecret(ctx, s) if err != nil { return fmt.Errorf("creating Alertmanager User Workload RBAC proxy tenancy Secret failed: %w", err) } @@ -100,7 +100,7 @@ func (t *AlertmanagerUserWorkloadTask) create(ctx context.Context) error { return fmt.Errorf("initializing Alertmanager User Workload RBAC proxy metric Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, rsm) + err = t.client.CreateOrUpdateSecret(ctx, rsm) if err != nil { return fmt.Errorf("creating Alertmanager User Workload RBAC proxy metric Secret failed: %w", err) } diff --git a/pkg/tasks/kubestatemetrics.go b/pkg/tasks/kubestatemetrics.go index d3ecc0a401..d1990b0c9f 100644 --- a/pkg/tasks/kubestatemetrics.go +++ b/pkg/tasks/kubestatemetrics.go @@ -70,7 +70,7 @@ func (t *KubeStateMetricsTask) Run(ctx context.Context) error { return fmt.Errorf("initializing kube-state-metrics RBAC proxy Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, rs) + err = t.client.CreateOrUpdateSecret(ctx, rs) if err != nil { return fmt.Errorf("creating kube-state-metrics RBAC proxy Secret failed: %w", err) } diff --git a/pkg/tasks/nodeexporter.go b/pkg/tasks/nodeexporter.go index a587ad239a..25071eecd7 100644 --- a/pkg/tasks/nodeexporter.go +++ b/pkg/tasks/nodeexporter.go @@ -80,7 +80,7 @@ func (t *NodeExporterTask) Run(ctx context.Context) error { return fmt.Errorf("intializing node-exporter rbac proxy secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, nes) + err = t.client.CreateOrUpdateSecret(ctx, nes) if err != nil { return fmt.Errorf("creating node-exporter rbac proxy secret failed: %w", err) } diff --git a/pkg/tasks/openshiftstatemetrics.go b/pkg/tasks/openshiftstatemetrics.go index 21377f828b..90173888a2 100644 --- a/pkg/tasks/openshiftstatemetrics.go +++ b/pkg/tasks/openshiftstatemetrics.go @@ -80,7 +80,7 @@ func (t *OpenShiftStateMetricsTask) Run(ctx context.Context) error { return fmt.Errorf("initializing openshift-state-metrics RBAC proxy Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, rs) + err = t.client.CreateOrUpdateSecret(ctx, rs) if err != nil { return fmt.Errorf("creating openshift-state-metrics RBAC proxy Secret failed: %w", err) } diff --git a/pkg/tasks/prometheusoperator.go b/pkg/tasks/prometheusoperator.go index 4ecb05e078..fb73622967 100644 --- a/pkg/tasks/prometheusoperator.go +++ b/pkg/tasks/prometheusoperator.go @@ -85,7 +85,7 @@ func (t *PrometheusOperatorTask) Run(ctx context.Context) error { return fmt.Errorf("initializing Prometheus Operator RBAC proxy Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, rs) + err = t.client.CreateOrUpdateSecret(ctx, rs) if err != nil { return fmt.Errorf("creating Prometheus Operator RBAC proxy Secret failed: %w", err) } diff --git a/pkg/tasks/thanos_querier.go b/pkg/tasks/thanos_querier.go index 7ec8b2f677..e0dbdebb16 100644 --- a/pkg/tasks/thanos_querier.go +++ b/pkg/tasks/thanos_querier.go @@ -73,7 +73,7 @@ func (t *ThanosQuerierTask) Run(ctx context.Context) error { return fmt.Errorf("initializing Thanos Querier RBAC proxy Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, rs) + err = t.client.CreateOrUpdateSecret(ctx, rs) if err != nil { return fmt.Errorf("creating Thanos Querier RBAC proxy Secret failed: %w", err) } @@ -83,7 +83,7 @@ func (t *ThanosQuerierTask) Run(ctx context.Context) error { return fmt.Errorf("initializing Thanos Querier RBAC proxy rules Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, rs) + err = t.client.CreateOrUpdateSecret(ctx, rs) if err != nil { return fmt.Errorf("creating Thanos Querier RBAC proxy rules Secret failed: %w", err) } @@ -93,7 +93,7 @@ func (t *ThanosQuerierTask) Run(ctx context.Context) error { return fmt.Errorf("initializing Thanos Querier RBAC proxy metrics Secret failed: %w", err) } - err = t.client.CreateIfNotExistSecret(ctx, rs) + err = t.client.CreateOrUpdateSecret(ctx, rs) if err != nil { return fmt.Errorf("creating Thanos Querier RBAC proxy metrics Secret failed: %w", err) } diff --git a/test/e2e/reconcile_objects_test.go b/test/e2e/reconcile_objects_test.go new file mode 100644 index 0000000000..2cd0752654 --- /dev/null +++ b/test/e2e/reconcile_objects_test.go @@ -0,0 +1,187 @@ +package e2e + +import ( + "context" + "encoding/json" + "fmt" + "slices" + "strings" + "testing" + "time" + + "github.com/openshift/cluster-monitoring-operator/test/e2e/framework" + "github.com/stretchr/testify/require" + v1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" +) + +// TestSecretsReconciliation tests whether the secrets created by the operator are reconciled correctly. These include: +// * unsynced secrets: secrets that are deployed by, but not synced by the operator, and, +// * synced secrets: secrets that are deployed by, and should be synced by the operator. + +// TODO: Exclude all secrets that are initially empty and populated by other operators. +func TestSecretsReconciliation(t *testing.T) { + // Create assets under both scenarios for us to work with. + setupUserWorkloadAssetsWithTeardownHook(t, f) + userWorkloadConfigMap := f.BuildUserWorkloadConfigMap(t, `alertmanager: + enabled: true +`) + f.MustCreateOrUpdateConfigMap(t, userWorkloadConfigMap) + defer f.MustDeleteConfigMap(t, userWorkloadConfigMap) + + f.AssertStatefulSetExistsAndRollout("alertmanager-user-workload", f.UserWorkloadMonitoringNs)(t) + f.AssertServiceExists("alertmanager-user-workload", f.UserWorkloadMonitoringNs)(t) + f.AssertSecretExists("alertmanager-user-workload", f.UserWorkloadMonitoringNs)(t) + + // List of secrets that should not be synced during operator's reconciliation. + unsyncedSecrets := []types.NamespacedName{ + { + Name: "alertmanager-main", + Namespace: f.Ns, + }, + { + Name: "alertmanager-user-workload", + Namespace: f.UserWorkloadMonitoringNs, + }, + { + Name: "thanos-ruler-user-workload-config", + Namespace: f.UserWorkloadMonitoringNs, + }, + { + Name: "thanos-ruler-user-workload-web-config", + Namespace: f.UserWorkloadMonitoringNs, + }, + } + + // Restore all unsynced secrets to their original state. + cleanup := func() { + for _, secret := range unsyncedSecrets { + gotSecret, err := f.KubeClient.CoreV1().Secrets(secret.Namespace).Get(context.Background(), secret.Name, metav1.GetOptions{}) + if errors.IsNotFound(err) { + continue + } + require.NoError(t, err) + data := gotSecret.Data + for k, v := range data { + data[k] = []byte(strings.TrimPrefix(string(v), t.Name())) + } + _, err = f.KubeClient.CoreV1().Secrets(secret.Namespace).Update(context.Background(), gotSecret, metav1.UpdateOptions{}) + require.NoError(t, err) + } + } + defer cleanup() + + var syncedSecrets []types.NamespacedName + secretsNS, err := f.KubeClient.CoreV1().Secrets(f.Ns).List(context.Background(), metav1.ListOptions{ + // Intentionally commented out as we want to fetch all secrets. + // LabelSelector: "app.kubernetes.io/managed-by=cluster-monitoring-operator", + }) + require.NoError(t, err) + + secretsUWMNS, err := f.KubeClient.CoreV1().Secrets(f.UserWorkloadMonitoringNs).List(context.Background(), metav1.ListOptions{ + // Intentionally commented out as we want to fetch all secrets. + // LabelSelector: "app.kubernetes.io/managed-by=cluster-monitoring-operator", + }) + require.NoError(t, err) + + for _, secret := range append(secretsNS.Items, secretsUWMNS.Items...) { + secretNamespacedName := types.NamespacedName{ + Name: secret.Name, + Namespace: secret.Namespace, + } + if slices.Contains(unsyncedSecrets, secretNamespacedName) { + continue + } + syncedSecrets = append(syncedSecrets, secretNamespacedName) + } + require.NotEmpty(t, syncedSecrets) + + secrets := append(syncedSecrets, unsyncedSecrets...) + + var filteredSecrets []types.NamespacedName + for _, secret := range secrets { + if strings.Contains(secret.Name, "tls") { + continue + } + filteredSecrets = append(filteredSecrets, secret) + } + + // Update the aforementioned secrets' data. + for _, secret := range filteredSecrets { + var gotSecret *v1.Secret + gotSecret, err = f.KubeClient.CoreV1().Secrets(secret.Namespace).Get(context.Background(), secret.Name, metav1.GetOptions{}) + require.NoError(t, err) + data := gotSecret.Data + for k, v := range data { + if isSecretDataJSONEncoded(gotSecret) { + var jsonData map[string]interface{} + err = json.Unmarshal(v, &jsonData) + require.NoError(t, err) + jsonData[t.Name()] = t.Name() + v, err = json.Marshal(jsonData) + require.NoError(t, err) + data[k] = v + } else { + data[k] = []byte(t.Name() + string(v)) + } + break + } + + _, err = f.KubeClient.CoreV1().Secrets(secret.Namespace).Update(context.Background(), gotSecret, metav1.UpdateOptions{}) + require.NoError(t, err) + } + + // Check for reconciliation of secrets. + for _, secret := range filteredSecrets { + // Check if the secrets were reconciled as expected. + if slices.Contains(syncedSecrets, secret) { + err = framework.Poll(10*time.Second, 5*time.Minute, func() error { + var updatedSecret *v1.Secret + updatedSecret, err = f.KubeClient.CoreV1().Secrets(secret.Namespace).Get(context.Background(), secret.Name, metav1.GetOptions{}) + if err != nil { + return err + } + + data := updatedSecret.Data + for _, v := range data { + if !isSecretDataJSONEncoded(updatedSecret) { + if strings.HasPrefix(string(v), t.Name()) { + return fmt.Errorf("secret %s has unexpected data: %v", secret.String(), string(v)) + } + return nil + } + + var jsonData map[string]interface{} + err = json.Unmarshal(v, &jsonData) + if err != nil { + return fmt.Errorf("failed to unmarshal JSON data in secret %s: %v", secret.String(), err) + } + if _, ok := jsonData[t.Name()]; ok { + return fmt.Errorf("secret %s does contains unexpected key %s", secret.String(), t.Name()) + } + } + + return nil + }) + + require.NoError(t, err) + } + + // Check if the secrets were reconciled unexpectedly. + if slices.Contains(unsyncedSecrets, secret) { + var updatedSecret *v1.Secret + updatedSecret, err = f.KubeClient.CoreV1().Secrets(secret.Namespace).Get(context.Background(), secret.Name, metav1.GetOptions{}) + require.NoError(t, err) + data := updatedSecret.Data + for _, v := range data { + require.False(t, strings.HasPrefix(string(v), t.Name()), fmt.Sprintf("secret %s was unexpectedly reconciled", secret.String())) + } + } + } +} + +func isSecretDataJSONEncoded(secret *v1.Secret) bool { + return secret.Type == v1.SecretTypeDockercfg || secret.Type == v1.SecretTypeDockerConfigJson +}