Add config option to enable the encryption of AWS EKS secrets (#2788)

Author: joneszc

src/_nebari/provider/cloud/amazon_web_services.py

@@ -2,6 +2,7 @@
 import os
 import re
 import time
+from dataclasses import dataclass
 from typing import Dict, List, Optional
 
 import boto3
@@ -121,6 +122,35 @@ def instances(region: str) -> Dict[str, str]:
     return {t: t for t in instance_types}
 
 
+@dataclass
+class Kms_Key_Info:
+    Arn: str
+    KeyUsage: str
+    KeySpec: str
+    KeyManager: str
+
+
+@functools.lru_cache()
+def kms_key_arns(region: str) -> Dict[str, Kms_Key_Info]:
+    """Return dict of available/enabled KMS key IDs and associated KeyMetadata for the AWS region."""
+    session = aws_session(region=region)
+    client = session.client("kms")
+    kms_keys = {}
+    # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kms/client/list_keys.html
+    for key in client.list_keys().get("Keys"):
+        key_id = key["KeyId"]
+        # https://boto3.amazonaws.com/v1/documentation/api/latest/reference/services/kms/client/describe_key.html#:~:text=Response%20Structure
+        key_data = client.describe_key(KeyId=key_id).get("KeyMetadata")
+        if key_data.get("Enabled"):
+            kms_keys[key_id] = Kms_Key_Info(
+                Arn=key_data.get("Arn"),
+                KeyUsage=key_data.get("KeyUsage"),
+                KeySpec=key_data.get("KeySpec"),
+                KeyManager=key_data.get("KeyManager"),
+            )
+    return kms_keys
+
+
 def aws_get_vpc_id(name: str, namespace: str, region: str) -> Optional[str]:
     """Return VPC ID for the EKS cluster namedd `{name}-{namespace}`."""
     cluster_name = f"{name}-{namespace}"

src/_nebari/stages/infrastructure/__init__.py

@@ -174,6 +174,7 @@ class AWSInputVars(schema.Base):
     eks_endpoint_access: Optional[
         Literal["private", "public", "public_and_private"]
     ] = "public"
+    eks_kms_arn: Optional[str] = None
     node_groups: List[AWSNodeGroupInputVars]
     availability_zones: List[str]
     vpc_cidr_block: str
@@ -490,6 +491,7 @@ class AmazonWebServicesProvider(schema.Base):
     eks_endpoint_access: Optional[
         Literal["private", "public", "public_and_private"]
     ] = "public"
+    eks_kms_arn: Optional[str] = None
     existing_subnet_ids: Optional[List[str]] = None
     existing_security_group_id: Optional[str] = None
     vpc_cidr_block: str = "10.10.0.0/16"
@@ -546,6 +548,42 @@ def _check_input(cls, data: Any) -> Any:
                         f"Amazon Web Services instance {node_group.instance} not one of available instance types={available_instances}"
                     )
 
+        # check if kms key is valid
+        available_kms_keys = amazon_web_services.kms_key_arns(data["region"])
+        if "eks_kms_arn" in data and data["eks_kms_arn"] is not None:
+            key_id = [
+                id for id in available_kms_keys.keys() if id in data["eks_kms_arn"]
+            ]
+            # Raise error if key_id is not found in available_kms_keys
+            if (
+                len(key_id) != 1
+                or available_kms_keys[key_id[0]].Arn != data["eks_kms_arn"]
+            ):
+                raise ValueError(
+                    f"Amazon Web Services KMS Key with ARN {data['eks_kms_arn']} not one of available/enabled keys={[v.Arn for v in available_kms_keys.values() if v.KeyManager=='CUSTOMER' and v.KeySpec=='SYMMETRIC_DEFAULT']}"
+                )
+            key_id = key_id[0]
+            # Raise error if key is not a customer managed key
+            if available_kms_keys[key_id].KeyManager != "CUSTOMER":
+                raise ValueError(
+                    f"Amazon Web Services KMS Key with ID {key_id} is not a customer managed key"
+                )
+            # Symmetric KMS keys with Encrypt and decrypt key-usage have the SYMMETRIC_DEFAULT key-spec
+            # EKS cluster encryption requires a Symmetric key that is set to encrypt and decrypt data
+            if available_kms_keys[key_id].KeySpec != "SYMMETRIC_DEFAULT":
+                if available_kms_keys[key_id].KeyUsage == "GENERATE_VERIFY_MAC":
+                    raise ValueError(
+                        f"Amazon Web Services KMS Key with ID {key_id} does not have KeyUsage set to 'Encrypt and decrypt' data"
+                    )
+                elif available_kms_keys[key_id].KeyUsage != "ENCRYPT_DECRYPT":
+                    raise ValueError(
+                        f"Amazon Web Services KMS Key with ID {key_id} is not of type Symmetric, and KeyUsage not set to 'Encrypt and decrypt' data"
+                    )
+                else:
+                    raise ValueError(
+                        f"Amazon Web Services KMS Key with ID {key_id} is not of type Symmetric"
+                    )
+
         return data
 
 
@@ -835,6 +873,7 @@ def input_vars(self, stage_outputs: Dict[str, Dict[str, Any]]):
                 name=self.config.escaped_project_name,
                 environment=self.config.namespace,
                 eks_endpoint_access=self.config.amazon_web_services.eks_endpoint_access,
+                eks_kms_arn=self.config.amazon_web_services.eks_kms_arn,
                 existing_subnet_ids=self.config.amazon_web_services.existing_subnet_ids,
                 existing_security_group_id=self.config.amazon_web_services.existing_security_group_id,
                 region=self.config.amazon_web_services.region,

src/_nebari/stages/infrastructure/template/aws/main.tf

@@ -99,6 +99,7 @@ module "kubernetes" {
 
   endpoint_public_access  = var.eks_endpoint_access == "private" ? false : true
   endpoint_private_access = var.eks_endpoint_access == "public" ? false : true
+  eks_kms_arn             = var.eks_kms_arn
   public_access_cidrs     = var.eks_public_access_cidrs
   permissions_boundary    = var.permissions_boundary
 }

src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/main.tf

@@ -14,8 +14,20 @@ resource "aws_eks_cluster" "main" {
     public_access_cidrs     = var.public_access_cidrs
   }
 
+  # Only set encryption_config if eks_kms_arn is not null
+  dynamic "encryption_config" {
+    for_each = var.eks_kms_arn != null ? [1] : []
+    content {
+      provider {
+        key_arn = var.eks_kms_arn
+      }
+      resources = ["secrets"]
+    }
+  }
+
   depends_on = [
     aws_iam_role_policy_attachment.cluster-policy,
+    aws_iam_role_policy_attachment.cluster_encryption,
   ]
 
   tags = merge({ Name = var.name }, var.tags)

src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/policy.tf

@@ -32,6 +32,33 @@ resource "aws_iam_role_policy_attachment" "cluster-policy" {
   role       = aws_iam_role.cluster.name
 }
 
+data "aws_iam_policy_document" "cluster_encryption" {
+  count = var.eks_kms_arn != null ? 1 : 0
+  statement {
+    actions = [
+      "kms:Encrypt",
+      "kms:Decrypt",
+      "kms:ListGrants",
+      "kms:DescribeKey"
+    ]
+    resources = [var.eks_kms_arn]
+  }
+}
+
+resource "aws_iam_policy" "cluster_encryption" {
+  count       = var.eks_kms_arn != null ? 1 : 0
+  name        = "${var.name}-eks-encryption-policy"
+  description = "IAM policy for EKS cluster encryption"
+  policy      = data.aws_iam_policy_document.cluster_encryption[count.index].json
+}
+
+# Grant the EKS Cluster role KMS permissions if a key-arn is specified
+resource "aws_iam_role_policy_attachment" "cluster_encryption" {
+  count      = var.eks_kms_arn != null ? 1 : 0
+  policy_arn = aws_iam_policy.cluster_encryption[count.index].arn
+  role       = aws_iam_role.cluster.name
+}
+
 # =======================================================
 # Kubernetes Node Group Policies
 # =======================================================

src/_nebari/stages/infrastructure/template/aws/modules/kubernetes/variables.tf

@@ -72,6 +72,12 @@ variable "endpoint_private_access" {
   default = false
 }
 
+variable "eks_kms_arn" {
+  description = "kms key arn for EKS cluster encryption_config"
+  type        = string
+  default     = null
+}
+
 variable "public_access_cidrs" {
   type    = list(string)
   default = ["0.0.0.0/0"]

src/_nebari/stages/infrastructure/template/aws/variables.tf

@@ -69,6 +69,12 @@ variable "eks_endpoint_private_access" {
   default = false
 }
 
+variable "eks_kms_arn" {
+  description = "kms key arn for EKS cluster encryption_config"
+  type        = string
+  default     = null
+}
+
 variable "eks_public_access_cidrs" {
   type    = list(string)
   default = ["0.0.0.0/0"]

tests/tests_unit/conftest.py

@@ -56,6 +56,18 @@ def _mock_return_value(return_value):
             "m5.xlarge": "m5.xlarge",
             "m5.2xlarge": "m5.2xlarge",
         },
+        "_nebari.provider.cloud.amazon_web_services.kms_key_arns": {
+            "xxxxxxxx-east-zzzz": {
+                "Arn": "arn:aws:kms:us-east-1:100000:key/xxxxxxxx-east-zzzz",
+                "KeyUsage": "ENCRYPT_DECRYPT",
+                "KeySpec": "SYMMETRIC_DEFAULT",
+            },
+            "xxxxxxxx-west-zzzz": {
+                "Arn": "arn:aws:kms:us-west-2:100000:key/xxxxxxxx-west-zzzz",
+                "KeyUsage": "ENCRYPT_DECRYPT",
+                "KeySpec": "SYMMETRIC_DEFAULT",
+            },
+        },
         # Azure
         "_nebari.provider.cloud.azure_cloud.kubernetes_versions": [
             "1.18",