Add automatic rolling deployment & option to re-create admission webhook certs

.github/workflows/amazon-cloudwatch-observability-integration-test.yaml

@@ -48,6 +48,10 @@ jobs:
           - appsignals-unsupported
           - webhooks-partially-enabled
           - webhooks-configured
+          - deployment-rolling-enabled
+          - deployment-rolling-disabled
+          - certificate-recreate-enabled
+          - certificate-recreate-disabled
     steps:
       - uses: actions/checkout@v3
 

charts/amazon-cloudwatch-observability/templates/_helpers.tpl

@@ -408,4 +408,32 @@ Get namespaceSelector value for admission webhooks
 {{- end -}}
 {{- end -}}
 
-
+{{/*
+Returns auto-generated certificate and CA for admission webhooks.
+*/}}
+{{- define "amazon-cloudwatch-observability.webhookCert" -}}
+{{- $tlsCrt := "" }}
+{{- $tlsKey := "" }}
+{{- $caCrt := "" }}
+{{- if .Values.admissionWebhooks.autoGenerateCert.enabled }}
+{{- $existingCert := ( lookup "v1" "Secret" .Release.Namespace (include "amazon-cloudwatch-observability.certificateSecretName" .) ) }}
+{{- if and (not .Values.admissionWebhooks.autoGenerateCert.recreate) $existingCert }}
+{{- $tlsCrt = index $existingCert "data" "tls.crt" }}
+{{- $tlsKey = index $existingCert "data" "tls.key" }}
+{{- $caCrt = index $existingCert "data" "ca.crt" }}
+{{- if not $caCrt }}
+{{- $existingWebhook := ( lookup "admissionregistration.k8s.io/v1" "MutatingWebhookConfiguration" "" (printf "%s-mutating-webhook-configuration" (include "amazon-cloudwatch-observability.name" .)) ) }}
+{{- $caCrt = (first $existingWebhook.webhooks).clientConfig.caBundle }}
+{{- end }}
+{{- else }}
+{{- $altNames := list ( printf "%s-webhook-service.%s" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) ( printf "%s-webhook-service.%s.svc" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) ( printf "%s-webhook-service.%s.svc.cluster.local" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) -}}
+{{- $ca := genCA ( printf "%s-ca" (include "amazon-cloudwatch-observability.name" .) ) ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) -}}
+{{- $cert := genSignedCert (include "amazon-cloudwatch-observability.name" .) nil $altNames ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}}
+{{- $tlsCrt = b64enc $cert.Cert }}
+{{- $tlsKey = b64enc $cert.Key }}
+{{- $caCrt = b64enc $ca.Cert }}
+{{- end }}
+{{- $result := dict "Cert" $tlsCrt "Key" $tlsKey "Ca" $caCrt }}
+{{- $result | toYaml }}
+{{- end }}
+{{- end }}

charts/amazon-cloudwatch-observability/templates/admission-webhooks/operator-webhook.yaml

@@ -1,7 +1,5 @@
 {{- if and (.Values.admissionWebhooks.autoGenerateCert.enabled) (not .Values.admissionWebhooks.certManager.enabled) (include "amazon-cloudwatch-observability.webhookEnabled" .) }}
-{{- $altNames := list ( printf "%s-webhook-service.%s" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) ( printf "%s-webhook-service.%s.svc" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) ( printf "%s-webhook-service.%s.svc.cluster.local" (include "amazon-cloudwatch-observability.name" .) .Release.Namespace ) -}}
-{{- $ca := genCA ( printf "%s-ca" (include "amazon-cloudwatch-observability.name" .) ) ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) -}}
-{{- $cert := genSignedCert (include "amazon-cloudwatch-observability.name" .) nil $altNames ( .Values.admissionWebhooks.autoGenerateCert.expiryDays | int ) $ca -}}
+{{- $cert := fromYaml (include "amazon-cloudwatch-observability.webhookCert" .) }}
 apiVersion: v1
 kind: Secret
 type: kubernetes.io/tls
@@ -11,8 +9,8 @@ metadata:
   name: {{ template "amazon-cloudwatch-observability.certificateSecretName" . }}
   namespace: {{ .Release.Namespace }}
 data:
-  tls.crt: {{ $cert.Cert | b64enc }}
-  tls.key: {{ $cert.Key | b64enc }}
+  tls.crt: {{ $cert.Cert }}
+  tls.key: {{ $cert.Key }}
 ---
 apiVersion: admissionregistration.k8s.io/v1
 kind: MutatingWebhookConfiguration
@@ -29,7 +27,7 @@ webhooks:
       name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
       path: /mutate-cloudwatch-aws-amazon-com-v1alpha1-instrumentation
-    caBundle: {{ $ca.Cert | b64enc }}
+    caBundle: {{ $cert.Ca }}
   failurePolicy: {{ .Values.admissionWebhooks.instrumentations.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
   name: minstrumentation.kb.io
   namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "instrumentations") }}
@@ -58,7 +56,7 @@ webhooks:
       name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
       path: /mutate-cloudwatch-aws-amazon-com-v1alpha1-amazoncloudwatchagent
-    caBundle: {{ $ca.Cert | b64enc }}
+    caBundle: {{ $cert.Ca }}
   failurePolicy: {{ .Values.admissionWebhooks.agents.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
   name: mamazoncloudwatchagent.kb.io
   namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "agents") }}
@@ -87,7 +85,7 @@ webhooks:
       name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
       path: /mutate-v1-pod
-    caBundle: {{ $ca.Cert | b64enc }}
+    caBundle: {{ $cert.Ca }}
   failurePolicy: {{ .Values.admissionWebhooks.pods.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
   name: mpod.kb.io
   namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "pods") }}
@@ -116,7 +114,7 @@ webhooks:
       name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
       path: /mutate-v1-namespace
-    caBundle: {{ $ca.Cert | b64enc }}
+    caBundle: {{ $cert.Ca }}
   failurePolicy: {{ .Values.admissionWebhooks.namespaces.failurePolicy | default .Values.admissionWebhooks.pods.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
   name: mnamespace.kb.io
   namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "namespaces") }}
@@ -145,7 +143,7 @@ webhooks:
       name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
       path: /mutate-v1-workload
-    caBundle: {{ $ca.Cert | b64enc }}
+    caBundle: {{ $cert.Ca }}
   failurePolicy: {{ .Values.admissionWebhooks.workloads.failurePolicy | default .Values.admissionWebhooks.pods.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
   name: mworkload.kb.io
   namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "workloads") }}
@@ -184,7 +182,7 @@ webhooks:
       name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
       path: /validate-cloudwatch-aws-amazon-com-v1alpha1-instrumentation
-    caBundle: {{ $ca.Cert | b64enc }}
+    caBundle: {{ $cert.Ca }}
   failurePolicy: {{ .Values.admissionWebhooks.instrumentations.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
   name: vinstrumentationcreateupdate.kb.io
   namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "instrumentations") }}
@@ -213,7 +211,7 @@ webhooks:
       name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
       path: /validate-cloudwatch-aws-amazon-com-v1alpha1-instrumentation
-    caBundle: {{ $ca.Cert | b64enc }}
+    caBundle: {{ $cert.Ca }}
   failurePolicy: Ignore
   name: vinstrumentationdelete.kb.io
   namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "instrumentations") }}
@@ -241,7 +239,7 @@ webhooks:
       name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
       path: /validate-cloudwatch-aws-amazon-com-v1alpha1-amazoncloudwatchagent
-    caBundle: {{ $ca.Cert | b64enc }}
+    caBundle: {{ $cert.Ca }}
   failurePolicy: {{ .Values.admissionWebhooks.agents.failurePolicy | default .Values.admissionWebhooks.failurePolicy }}
   name: vamazoncloudwatchagentcreateupdate.kb.io
   namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "agents") }}
@@ -270,7 +268,7 @@ webhooks:
       name: {{ template "amazon-cloudwatch-observability.webhookServiceName" . }}
       namespace: {{ .Release.Namespace }}
       path: /validate-cloudwatch-aws-amazon-com-v1alpha1-amazoncloudwatchagent
-    caBundle: {{ $ca.Cert | b64enc }}
+    caBundle: {{ $cert.Ca }}
   failurePolicy: Ignore
   name: vamazoncloudwatchagentdelete.kb.io
   namespaceSelector: {{ include "amazon-cloudwatch-observability.namespaceSelector" (list . "agents") }}

charts/amazon-cloudwatch-observability/templates/operator-deployment.yaml

@@ -18,6 +18,9 @@ spec:
         {{- if .Values.manager.podAnnotations }}
         {{- include "amazon-cloudwatch-observability.podAnnotations" . | nindent 8 }}
         {{- end }}
+        {{- if .Values.manager.rolling }}
+        rollme: {{ randAlphaNum 5 | quote }}
+        {{- end }}
       labels:
         app.kubernetes.io/name: {{ template "amazon-cloudwatch-observability.name" . }}
         control-plane: controller-manager

charts/amazon-cloudwatch-observability/values.yaml

@@ -1208,6 +1208,8 @@ manager:
   affinity: {}
   nodeSelector:
     kubernetes.io/os: linux
+  # Enable automatic rolling by forcing a deployment spec change
+  rolling: false
 ## Admission webhooks make sure only requests with correctly formatted rules will get into the Operator.
 admissionWebhooks:
   create: true
@@ -1256,6 +1258,7 @@ admissionWebhooks:
   autoGenerateCert:
     enabled: true
     expiryDays: 3650 # 10 years
+    recreate: true
   ## TLS Certificate Option 2: Use certManager to generate self-signed certificate.
   ## certManager must be enabled. If enabled, it takes precedence over option 1.
   certManager:

integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/certificate-recreate-disabled/main.tf

@@ -0,0 +1,31 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+module "base" {
+  source           = "../.."
+  helm_dir         = var.helm_dir
+  helm_values_file = "${path.module}/values.yaml"
+}
+
+variable "helm_dir" {
+  type    = string
+  default = "../../../../../../charts/amazon-cloudwatch-observability"
+}
+
+resource "null_resource" "validator" {
+  depends_on = [module.base.helm_release]
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      go test ${var.test_dir} -v -run=TestCertificateRecreateDisabled_Save
+      helm upgrade --wait --create-namespace --namespace amazon-cloudwatch amazon-cloudwatch-observability ${var.helm_dir} -f ${path.module}/values.yaml
+      go test ${var.test_dir} -v -run=TestCertificateRecreateDisabled_Compare
+    EOT
+  }
+}
+
+variable "test_dir" {
+  type    = string
+  default = "../../../../validations/minikube/scenarios"
+}
+

integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/certificate-recreate-disabled/values.yaml

@@ -0,0 +1,6 @@
+region: us-west-2
+clusterName: minikube
+
+admissionWebhooks:
+  autoGenerateCert:
+    recreate: false

integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/certificate-recreate-enabled/main.tf

@@ -0,0 +1,30 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+module "base" {
+  source           = "../.."
+  helm_dir         = var.helm_dir
+  helm_values_file = "${path.module}/values.yaml"
+}
+
+variable "helm_dir" {
+  type    = string
+  default = "../../../../../../charts/amazon-cloudwatch-observability"
+}
+
+resource "null_resource" "validator" {
+  depends_on = [module.base.helm_release]
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      go test ${var.test_dir} -v -run=TestCertificateRecreateEnabled_Save
+      helm upgrade --wait --create-namespace --namespace amazon-cloudwatch amazon-cloudwatch-observability ${var.helm_dir} -f ${path.module}/values.yaml
+      go test ${var.test_dir} -v -run=TestCertificateRecreateEnabled_Compare
+    EOT
+  }
+}
+
+variable "test_dir" {
+  type    = string
+  default = "../../../../validations/minikube/scenarios"
+}

integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/certificate-recreate-enabled/values.yaml

@@ -0,0 +1,6 @@
+region: us-west-2
+clusterName: minikube
+
+admissionWebhooks:
+  autoGenerateCert:
+    recreate: true

integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/deployment-rolling-disabled/main.tf

@@ -0,0 +1,28 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+module "base" {
+  source           = "../.."
+  helm_dir         = var.helm_dir
+  helm_values_file = "${path.module}/values.yaml"
+}
+
+variable "helm_dir" {
+  type    = string
+  default = "../../../../../../charts/amazon-cloudwatch-observability"
+}
+
+resource "null_resource" "validator" {
+  depends_on = [module.base.helm_release]
+
+  provisioner "local-exec" {
+    command = "go test ${var.test_dir} -v -run=TestDeploymentRollingDisabled"
+  }
+}
+
+variable "test_dir" {
+  type    = string
+  default = "../../../../validations/minikube/scenarios"
+}
+
+

integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/deployment-rolling-disabled/values.yaml

@@ -0,0 +1,5 @@
+region: us-west-2
+clusterName: minikube
+
+manager:
+  rolling: false

integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/deployment-rolling-enabled/main.tf

@@ -0,0 +1,33 @@
+// Copyright Amazon.com, Inc. or its affiliates. All Rights Reserved.
+// SPDX-License-Identifier: MIT
+
+module "base" {
+  source           = "../.."
+  helm_dir         = var.helm_dir
+  helm_values_file = "${path.module}/values.yaml"
+}
+
+variable "helm_dir" {
+  type    = string
+  default = "../../../../../../charts/amazon-cloudwatch-observability"
+}
+
+resource "null_resource" "validator" {
+  depends_on = [module.base.helm_release]
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      go test ${var.test_dir} -v -run=TestDeploymentRollingEnabled_Save
+      helm upgrade --wait --create-namespace --namespace amazon-cloudwatch amazon-cloudwatch-observability ${var.helm_dir} -f ${path.module}/values.yaml
+      go test ${var.test_dir} -v -run=TestDeploymentRollingEnabled_Compare
+    EOT
+  }
+}
+
+variable "test_dir" {
+  type    = string
+  default = "../../../../validations/minikube/scenarios"
+}
+
+
+

integration-tests/amazon-cloudwatch-observability/terraform/minikube/scenarios/deployment-rolling-enabled/values.yaml

@@ -0,0 +1,6 @@
+region: us-west-2
+clusterName: minikube
+
+manager:
+  rolling: true
+

integration-tests/amazon-cloudwatch-observability/validations/minikube/common.go

@@ -15,7 +15,7 @@ import (
 
 const (
 	Namespace    = "amazon-cloudwatch"
-	operatorName = "amazon-cloudwatch-observability-controller-manager"
+	OperatorName = "amazon-cloudwatch-observability-controller-manager"
 
 	WebhookName                              = "amazon-cloudwatch-observability-mutating-webhook-configuration"
 	WebhookPathMutateInstrumentation         = "/mutate-cloudwatch-aws-amazon-com-v1alpha1-instrumentation"
@@ -37,7 +37,7 @@ func ValidateOperatorAutoMonitorConfig(t *testing.T, expectedConfig map[string]i
 	// Find the operator deployment by name
 	var deployment *appsV1.Deployment
 	for i := range deployments.Items {
-		if deployments.Items[i].Name == operatorName {
+		if deployments.Items[i].Name == OperatorName {
 			deployment = &deployments.Items[i]
 			break
 		}