Enable updating encryption config for EKS Clusters (#104)

a-hilaly · web-flow · commit c03b66eb510a · 2024-03-07T00:57:31.000Z
This commits adds the capability fo rthe eks-controller to update the
encryption configuration for an EKS Cluster. Only clusters that didn't
receive a encryption key on create are able to be updated.

If a user tries to modify or delete an existing encryption configuraiton
the controller will set a terminal condition stating that it's an
impossible operation.

Signed-off-by: Amine Hilaly &lt;hilalyamine@gmail.com&gt;

By submitting this pull request, I confirm that my contribution is made under the terms of the Apache 2.0 license.
diff --git a/apis/v1alpha1/ack-generate-metadata.yaml b/apis/v1alpha1/ack-generate-metadata.yaml
@@ -7,7 +7,7 @@ api_directory_checksum: d960a9f06b58cc445e5ab21fb26ee6d92c441374
 api_version: v1alpha1
 aws_sdk_go_version: v1.49.13
 generator_config_info:
-  file_checksum: ac8c0c6f258d7b552ccbaf81401e6c511de3373e
+  file_checksum: d4bad3acfd8ddb3923869d78c65384973ff25151
   original_file_name: generator.yaml
 last_modification:
   reason: API generation
diff --git a/apis/v1alpha1/generator.yaml b/apis/v1alpha1/generator.yaml
@@ -73,10 +73,10 @@ resources:
         priority: 1
   Cluster:
     fields:
-      EncryptionProvider.Provider.KeyArn:
+      EncryptionConfig.Provider.KeyArn:
         references:
           service_name: kms
-          resources: Key
+          resource: Key
           path: Status.ACKResourceMetadata.ARN
       RoleArn:
         references:
diff --git a/apis/v1alpha1/types.go b/apis/v1alpha1/types.go
diff --git a/apis/v1alpha1/zz_generated.deepcopy.go b/apis/v1alpha1/zz_generated.deepcopy.go
diff --git a/cmd/controller/main.go b/cmd/controller/main.go
diff --git a/config/crd/bases/eks.services.k8s.aws_clusters.yaml b/config/crd/bases/eks.services.k8s.aws_clusters.yaml
@@ -88,6 +88,18 @@ spec:
                       properties:
                         keyARN:
                           type: string
+                        keyRef:
+                          description: Reference field for KeyARN
+                          properties:
+                            from:
+                              description: |-
+                                AWSResourceReference provides all the values necessary to reference another
+                                k8s resource for finding the identifier(Id/ARN/Name)
+                              properties:
+                                name:
+                                  type: string
+                              type: object
+                          type: object
                       type: object
                     resources:
                       items:
diff --git a/config/rbac/cluster-role-controller.yaml b/config/rbac/cluster-role-controller.yaml
@@ -192,6 +192,20 @@ rules:
   verbs:
   - get
   - list
+- apiGroups:
+  - kms.services.k8s.aws
+  resources:
+  - keys
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - kms.services.k8s.aws
+  resources:
+  - keys/status
+  verbs:
+  - get
+  - list
 - apiGroups:
   - services.k8s.aws
   resources:
diff --git a/generator.yaml b/generator.yaml
@@ -73,10 +73,10 @@ resources:
         priority: 1
   Cluster:
     fields:
-      EncryptionProvider.Provider.KeyArn:
+      EncryptionConfig.Provider.KeyArn:
         references:
           service_name: kms
-          resources: Key
+          resource: Key
           path: Status.ACKResourceMetadata.ARN
       RoleArn:
         references:
diff --git a/go.mod b/go.mod
@@ -7,6 +7,7 @@ toolchain go1.21.5
 require (
 	github.com/aws-controllers-k8s/ec2-controller v1.1.0
 	github.com/aws-controllers-k8s/iam-controller v1.3.1
+	github.com/aws-controllers-k8s/kms-controller v1.0.9
 	github.com/aws-controllers-k8s/runtime v0.30.0
 	github.com/aws/aws-sdk-go v1.49.13
 	github.com/go-logr/logr v1.4.1
diff --git a/go.sum b/go.sum
@@ -2,6 +2,8 @@ github.com/aws-controllers-k8s/ec2-controller v1.1.0 h1:SSutBopBi2nYWqNNZfkVgv7k
 github.com/aws-controllers-k8s/ec2-controller v1.1.0/go.mod h1:PvsQehgncHgcu9FiY13M45+GkVsKI98g7G83SrgH7vY=
 github.com/aws-controllers-k8s/iam-controller v1.3.1 h1:/3yH3tAtSVAAt2ulIqsmutjWXhbKWXSaEZ1pph358GE=
 github.com/aws-controllers-k8s/iam-controller v1.3.1/go.mod h1:7nZzMtEN8xEL5fYhL9FKkBhqoP4QMmMp5x5dXDGwfYM=
+github.com/aws-controllers-k8s/kms-controller v1.0.9 h1:GZHSnuZBoWp9r6RaJ3siyDn5BRhDuaZJXtdBKeAiLSw=
+github.com/aws-controllers-k8s/kms-controller v1.0.9/go.mod h1:Pnz0d5sly7dUgmYMDJWSRIKASOujJFi/b8N2q1qCLqU=
 github.com/aws-controllers-k8s/runtime v0.30.0 h1:AibYRdi/7xUA3t8BA0u8g+J+OioaTAT6R4Vq8hxLiYw=
 github.com/aws-controllers-k8s/runtime v0.30.0/go.mod h1:Pv1ozlUaO11KO2mwPN/HzhAtZ70ZDE9UP24mjsbkul0=
 github.com/aws/aws-sdk-go v1.49.13 h1:f4mGztsgnx2dR9r8FQYa9YW/RsKb+N7bgef4UGrOW1Y=
diff --git a/helm/crds/eks.services.k8s.aws_clusters.yaml b/helm/crds/eks.services.k8s.aws_clusters.yaml
@@ -88,6 +88,18 @@ spec:
                       properties:
                         keyARN:
                           type: string
+                        keyRef:
+                          description: Reference field for KeyARN
+                          properties:
+                            from:
+                              description: |-
+                                AWSResourceReference provides all the values necessary to reference another
+                                k8s resource for finding the identifier(Id/ARN/Name)
+                              properties:
+                                name:
+                                  type: string
+                              type: object
+                          type: object
                       type: object
                     resources:
                       items:
diff --git a/helm/templates/_helpers.tpl b/helm/templates/_helpers.tpl
@@ -238,6 +238,20 @@ rules:
   verbs:
   - get
   - list
+- apiGroups:
+  - kms.services.k8s.aws
+  resources:
+  - keys
+  verbs:
+  - get
+  - list
+- apiGroups:
+  - kms.services.k8s.aws
+  resources:
+  - keys/status
+  verbs:
+  - get
+  - list
 - apiGroups:
   - services.k8s.aws
   resources:
diff --git a/pkg/resource/cluster/hook.go b/pkg/resource/cluster/hook.go
@@ -24,6 +24,7 @@ import (
 	ackerr "github.com/aws-controllers-k8s/runtime/pkg/errors"
 	ackrequeue "github.com/aws-controllers-k8s/runtime/pkg/requeue"
 	ackrtlog "github.com/aws-controllers-k8s/runtime/pkg/runtime/log"
+	"github.com/aws/aws-sdk-go/aws"
 	svcsdk "github.com/aws/aws-sdk-go/service/eks"
 	corev1 "k8s.io/api/core/v1"
 
@@ -247,6 +248,45 @@ func (rm *resourceManager) customUpdate(
 		return returnClusterUpdating(updatedRes)
 	}
 
+	if delta.DifferentAt("Spec.EncryptionConfig") {
+		// Set a terminal condition if the observed cluster has encryption
+		// config and the desired cluster does not.
+		if len(latest.ko.Spec.EncryptionConfig) > 0 && len(desired.ko.Spec.EncryptionConfig) == 0 {
+			msg := "Encryption configuration cannot be removed from an existing cluster"
+			ackcondition.SetTerminal(updatedRes, corev1.ConditionTrue, &msg, nil)
+			return updatedRes, nil
+		}
+		// Set a terminal condition if the user tries to patch the encryption
+		// config of an existing cluster.
+		if len(latest.ko.Spec.EncryptionConfig) == 1 && len(desired.ko.Spec.EncryptionConfig) == 1 {
+			msg := "Encryption configuration cannot be updated"
+			ackcondition.SetTerminal(updatedRes, corev1.ConditionTrue, &msg, nil)
+			return updatedRes, nil
+		}
+		// Set a terminal condition if the user tries to add a second encryption
+		// config to an existing cluster.
+		if len(latest.ko.Spec.EncryptionConfig) == 0 && len(desired.ko.Spec.EncryptionConfig) > 1 {
+			msg := "Only one encryption configuration is allowed"
+			ackcondition.SetTerminal(updatedRes, corev1.ConditionTrue, &msg, nil)
+			return updatedRes, nil
+		}
+
+		if err := rm.associateEncryptionConfig(ctx, desired); err != nil {
+			awserr, ok := ackerr.AWSError(err)
+			// Check to see if we've raced an async update call and need to
+			// requeue
+			if ok && awserr.Code() == "ResourceInUseException" {
+				return nil, requeueAfterAsyncUpdate()
+			}
+
+			return nil, err
+		}
+		// This doesn't reflect the actual status of the cluster, so we have to explicitly
+		// requeue and set the status to updating.
+		updatedRes.ko.Status.Status = aws.String(string(svcsdk.ClusterStatusUpdating))
+		return returnClusterUpdating(updatedRes)
+	}
+
 	if delta.DifferentAt("Spec.Version") {
 		if err := rm.updateVersion(ctx, desired, latest); err != nil {
 			awserr, ok := ackerr.AWSError(err)
@@ -412,3 +452,37 @@ func (rm *resourceManager) listNodegroups(
 
 	return nodes, nil
 }
+
+// associateEncryptionConfig associates the encryption configuration with the
+// cluster.
+func (rm *resourceManager) associateEncryptionConfig(
+	ctx context.Context,
+	r *resource,
+) (err error) {
+	rlog := ackrtlog.FromContext(ctx)
+	exit := rlog.Trace("rm.updateEncryptionConfiguration")
+	defer func() { exit(err) }()
+
+	input := &svcsdk.AssociateEncryptionConfigInput{
+		ClusterName: r.ko.Spec.Name,
+		EncryptionConfig: []*svcsdk.EncryptionConfig{
+			{
+				// Being it means that we already have a single encryption config
+				// in the spec. So we can safely assume that the first element is
+				// the only one.
+				Resources: r.ko.Spec.EncryptionConfig[0].Resources,
+				Provider: &svcsdk.Provider{
+					KeyArn: r.ko.Spec.EncryptionConfig[0].Provider.KeyARN,
+				},
+			},
+		},
+	}
+
+	_, err = rm.sdkapi.AssociateEncryptionConfigWithContext(ctx, input)
+	rm.metrics.RecordAPICall("UPDATE", "AssociateEncryptionConfig", err)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
diff --git a/pkg/resource/cluster/references.go b/pkg/resource/cluster/references.go
diff --git a/test/e2e/tests/test_cluster.py b/test/e2e/tests/test_cluster.py
