feat: Add policy for custom tags on EKS Auto Mode, validate examples

Author: bryantbiggs

README.md

@@ -370,12 +370,14 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
 | [aws_iam_openid_connect_provider.oidc_provider](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_openid_connect_provider) | resource |
 | [aws_iam_policy.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_policy.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
+| [aws_iam_policy.eks_auto_custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_policy) | resource |
 | [aws_iam_role.eks_auto](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role) | resource |
 | [aws_iam_role_policy_attachment.additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.cluster_encryption](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.eks_auto](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.eks_auto_additional](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
+| [aws_iam_role_policy_attachment.eks_auto_custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_iam_role_policy_attachment.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/iam_role_policy_attachment) | resource |
 | [aws_security_group.cluster](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
 | [aws_security_group.node](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/security_group) | resource |
@@ -386,6 +388,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
 | [aws_eks_addon_version.this](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/eks_addon_version) | data source |
 | [aws_iam_policy_document.assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.cni_ipv6_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
+| [aws_iam_policy_document.eks_auto_custom](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_policy_document.node_assume_role_policy](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document) | data source |
 | [aws_iam_session_context.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_session_context) | data source |
 | [aws_partition.current](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/partition) | data source |
@@ -452,6 +455,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
 | <a name="input_enable_efa_support"></a> [enable\_efa\_support](#input\_enable\_efa\_support) | Determines whether to enable Elastic Fabric Adapter (EFA) support | `bool` | `false` | no |
 | <a name="input_enable_irsa"></a> [enable\_irsa](#input\_enable\_irsa) | Determines whether to create an OpenID Connect Provider for EKS to enable IRSA | `bool` | `true` | no |
 | <a name="input_enable_kms_key_rotation"></a> [enable\_kms\_key\_rotation](#input\_enable\_kms\_key\_rotation) | Specifies whether key rotation is enabled | `bool` | `true` | no |
+| <a name="input_enable_node_custom_tags_permissions"></a> [enable\_node\_custom\_tags\_permissions](#input\_enable\_node\_custom\_tags\_permissions) | Determines whether to enable permissions for custom tags for the EKS Auto node IAM role | `bool` | `true` | no |
 | <a name="input_enable_security_groups_for_pods"></a> [enable\_security\_groups\_for\_pods](#input\_enable\_security\_groups\_for\_pods) | Determines whether to add the necessary IAM permission policy for security groups for pods | `bool` | `true` | no |
 | <a name="input_fargate_profile_defaults"></a> [fargate\_profile\_defaults](#input\_fargate\_profile\_defaults) | Map of Fargate Profile default configurations | `any` | `{}` | no |
 | <a name="input_fargate_profiles"></a> [fargate\_profiles](#input\_fargate\_profiles) | Map of Fargate Profile definitions to create | `any` | `{}` | no |
@@ -479,6 +483,7 @@ We are grateful to the community for contributing bugfixes and improvements! Ple
 | <a name="input_node_iam_role_name"></a> [node\_iam\_role\_name](#input\_node\_iam\_role\_name) | Name to use on the EKS Auto node IAM role created | `string` | `null` | no |
 | <a name="input_node_iam_role_path"></a> [node\_iam\_role\_path](#input\_node\_iam\_role\_path) | The EKS Auto node IAM role path | `string` | `null` | no |
 | <a name="input_node_iam_role_permissions_boundary"></a> [node\_iam\_role\_permissions\_boundary](#input\_node\_iam\_role\_permissions\_boundary) | ARN of the policy that is used to set the permissions boundary for the EKS Auto node IAM role | `string` | `null` | no |
+| <a name="input_node_iam_role_policy_statements"></a> [node\_iam\_role\_policy\_statements](#input\_node\_iam\_role\_policy\_statements) | A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed | `any` | `[]` | no |
 | <a name="input_node_iam_role_tags"></a> [node\_iam\_role\_tags](#input\_node\_iam\_role\_tags) | A map of additional tags to add to the EKS Auto node IAM role created | `map(string)` | `{}` | no |
 | <a name="input_node_iam_role_use_name_prefix"></a> [node\_iam\_role\_use\_name\_prefix](#input\_node\_iam\_role\_use\_name\_prefix) | Determines whether the EKS Auto node IAM role name (`node_iam_role_name`) is used as a prefix | `bool` | `true` | no |
 | <a name="input_node_security_group_additional_rules"></a> [node\_security\_group\_additional\_rules](#input\_node\_security\_group\_additional\_rules) | List of additional security group rules to add to the node security group created. Set `source_cluster_security_group = true` inside rules to set the `cluster_security_group` as source | `any` | `{}` | no |

examples/eks-hybrid-nodes/ami/amazon-eks-ubuntu.pkr.hcl

@@ -296,7 +296,8 @@ build {
 
       "snap install aws-cli --classic",
       "snap switch --channel=candidate amazon-ssm-agent",
-      "curl -OL https://hybrid-assets.eks.amazonaws.com/releases/latest/bin/linux/amd64/nodeadm /usr/bin/",
+      "curl -OL 'https://hybrid-assets.eks.amazonaws.com/releases/latest/bin/linux/amd64/nodeadm'",
+      "mv nodeadm /usr/bin/nodeadm",
       "chmod +x /usr/bin/nodeadm",
       "nodeadm install ${var.eks_version} --credential-provider ${var.credential_provider}",
     ]

examples/eks-hybrid-nodes/main.tf

@@ -42,34 +42,27 @@ module "eks" {
   cluster_endpoint_public_access           = true
   enable_cluster_creator_admin_permissions = true
 
-  cluster_security_group_additional_rules = {
-    hybrid-all = {
-      cidr_blocks = [local.remote_network_cidr]
-      description = "Allow all HTTPS traffic from remote node/pod network"
-      from_port   = 443
-      to_port     = 443
-      protocol    = "tcp"
-      type        = "ingress"
-    }
+  cluster_addons = {
+    coredns                = {}
+    eks-pod-identity-agent = {}
+    kube-proxy             = {}
   }
 
-  node_security_group_additional_rules = {
+  create_node_security_group = false
+  cluster_security_group_additional_rules = {
     hybrid-all = {
       cidr_blocks = [local.remote_network_cidr]
       description = "Allow all traffic from remote node/pod network"
-      from_port   = "-1"
-      to_port     = "-1"
+      from_port   = 0
+      to_port     = 0
       protocol    = "all"
       type        = "ingress"
     }
   }
 
-  # EKS Addons
-  cluster_addons = {
-    coredns                = {}
-    eks-pod-identity-agent = {}
-    kube-proxy             = {}
-    vpc-cni                = {}
+  cluster_compute_config = {
+    enabled    = true
+    node_pools = ["system"]
   }
 
   access_entries = {
@@ -91,16 +84,6 @@ module "eks" {
     }
   }
 
-  eks_managed_node_groups = {
-    default = {
-      instance_types = ["m6i.large"]
-
-      min_size     = 2
-      max_size     = 5
-      desired_size = 2
-    }
-  }
-
   tags = local.tags
 }
 

examples/eks-hybrid-nodes/remote.tf

@@ -71,9 +71,13 @@ resource "local_file" "join" {
     EOF
 
     # Use SCP/SSH to execute commands on the remote host
-    scp -i ${local_file.key_pem.filename} nodeConfig.yaml ubuntu@${aws_instance.hybrid_node.public_ip}:/home/ubuntu/nodeConfig.yaml
-    ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node.public_ip} sudo nodeadm init -c file://nodeConfig.yaml
-    ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node.public_ip} sudo systemctl daemon-reload
+    scp -i ${local_file.key_pem.filename} nodeConfig.yaml ubuntu@${aws_instance.hybrid_node["one"].public_ip}:/home/ubuntu/nodeConfig.yaml
+    ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node["one"].public_ip} sudo nodeadm init -c file://nodeConfig.yaml
+    ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node["one"].public_ip} sudo systemctl daemon-reload
+
+    scp -i ${local_file.key_pem.filename} nodeConfig.yaml ubuntu@${aws_instance.hybrid_node["two"].public_ip}:/home/ubuntu/nodeConfig.yaml
+    ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node["two"].public_ip} sudo nodeadm init -c file://nodeConfig.yaml
+    ssh -n -i ${local_file.key_pem.filename} ubuntu@${aws_instance.hybrid_node["two"].public_ip} sudo systemctl daemon-reload
 
     # Clean up
     rm nodeConfig.yaml
@@ -85,14 +89,16 @@ data "aws_ami" "hybrid_node" {
   provider = aws.remote
 
   most_recent = true
-  name_regex  = "amazon-eks-ubuntu-${local.cluster_version}-amd64-*"
+  name_regex  = "eks-hybrid-ubuntu-${local.cluster_version}-amd64-*"
   owners      = ["self"]
 }
 
 # Demonstration only - AWS EC2 instances are not supported for EKS Hybrid nodes
 resource "aws_instance" "hybrid_node" {
   provider = aws.remote
 
+  for_each = { one = 0, two = 1 }
+
   ami                         = data.aws_ami.hybrid_node.id
   associate_public_ip_address = true
   instance_type               = "m5.large"
@@ -103,11 +109,11 @@ resource "aws_instance" "hybrid_node" {
   }
 
   vpc_security_group_ids = [aws_security_group.remote_node.id]
-  subnet_id              = element(module.remote_node_vpc.public_subnets, 0)
+  subnet_id              = element(module.remote_node_vpc.public_subnets, each.value)
 
   tags = merge(
     local.tags,
-    { Name = "hybrid-node" }
+    { Name = "hybrid-node-${each.key}" }
   )
 }
 
@@ -141,12 +147,10 @@ resource "aws_vpc_security_group_ingress_rule" "remote_node" {
     cluster-all = {
       description = "Allow all traffic from cluster network"
       cidr_ipv4   = module.vpc.vpc_cidr_block
-      from_port   = "-1"
       ip_protocol = "all"
     }
     remote-all = {
       description                  = "Allow all traffic from within the remote network itself"
-      from_port                    = "-1"
       ip_protocol                  = "all"
       referenced_security_group_id = aws_security_group.remote_node.id
     }
@@ -179,10 +183,7 @@ resource "aws_vpc_security_group_egress_rule" "remote_node" {
     all = {
       description = "Allow all egress"
       cidr_ipv4   = "0.0.0.0/0"
-      description = "All"
-      from_port   = "-1"
       ip_protocol = "all"
-      to_port     = "-1"
     }
   }
 
@@ -207,21 +208,14 @@ resource "helm_release" "cilium" {
   name       = "cilium"
   repository = "https://helm.cilium.io/"
   chart      = "cilium"
-  version    = "1.15.10"
+  version    = "1.16.4"
   namespace  = "kube-system"
   wait       = false
 
   values = [
     <<-EOT
-      affinity:
-        nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-              - matchExpressions:
-                - key: eks.amazonaws.com/compute-type
-                  operator: In
-                  values:
-                    - hybrid
+      nodeSelector:
+        eks.amazonaws.com/compute-type: hybrid
       ipam:
         mode: cluster-pool
         operator:

main.tf

@@ -691,6 +691,8 @@ resource "aws_eks_identity_provider_config" "this" {
 locals {
   create_node_iam_role = local.create && var.create_node_iam_role && local.auto_mode_nodepools_enabled
   node_iam_role_name   = coalesce(var.node_iam_role_name, "${var.cluster_name}-eks-auto")
+
+  create_node_iam_role_custom_policy = local.create_node_iam_role && (var.enable_node_custom_tags_permissions || length(var.node_iam_role_policy_statements) > 0)
 }
 
 data "aws_iam_policy_document" "node_assume_role_policy" {
@@ -742,3 +744,157 @@ resource "aws_iam_role_policy_attachment" "eks_auto_additional" {
   policy_arn = each.value
   role       = aws_iam_role.eks_auto[0].name
 }
+
+resource "aws_iam_role_policy_attachment" "eks_auto_custom" {
+  count = local.create_node_iam_role_custom_policy ? 1 : 0
+
+  policy_arn = aws_iam_policy.eks_auto_custom[0].arn
+  role       = aws_iam_role.eks_auto[0].name
+}
+
+data "aws_iam_policy_document" "eks_auto_custom" {
+  count = local.create_node_iam_role_custom_policy ? 1 : 0
+
+  dynamic "statement" {
+    for_each = var.enable_node_custom_tags_permissions ? [1] : []
+
+    content {
+      sid = "Compute"
+      actions = [
+        "ec2:CreateFleet",
+        "ec2:RunInstances",
+        "ec2:CreateLaunchTemplate",
+      ]
+      resources = ["*"]
+
+      condition {
+        test     = "StringEquals"
+        variable = "aws:RequestTag/eks:eks-cluster-name"
+        values   = ["$${aws:PrincipalTag/eks:eks-cluster-name}"]
+      }
+
+      condition {
+        test     = "StringLike"
+        variable = "aws:RequestTag/eks:kubernetes-node-class-name"
+        values   = ["*"]
+      }
+
+      condition {
+        test     = "StringLike"
+        variable = "aws:RequestTag/eks:kubernetes-node-pool-name"
+        values   = ["*"]
+      }
+    }
+  }
+
+  dynamic "statement" {
+    for_each = var.enable_node_custom_tags_permissions ? [1] : []
+
+    content {
+      sid = "Storage"
+      actions = [
+        "ec2:CreateVolume",
+        "ec2:CreateSnapshot",
+      ]
+      resources = [
+        "arn:${local.partition}:ec2:*:*:volume/*",
+        "arn:${local.partition}:ec2:*:*:snapshot/*",
+      ]
+
+      condition {
+        test     = "StringEquals"
+        variable = "aws:RequestTag/eks:eks-cluster-name"
+        values   = ["$${aws:PrincipalTag/eks:eks-cluster-name}"]
+      }
+    }
+  }
+
+  dynamic "statement" {
+    for_each = var.enable_node_custom_tags_permissions ? [1] : []
+
+    content {
+      sid       = "Networking"
+      actions   = ["ec2:CreateNetworkInterface"]
+      resources = ["*"]
+
+      condition {
+        test     = "StringEquals"
+        variable = "aws:RequestTag/eks:eks-cluster-name"
+        values   = ["$${aws:PrincipalTag/eks:eks-cluster-name}"]
+      }
+
+      condition {
+        test     = "StringEquals"
+        variable = "aws:RequestTag/eks:kubernetes-cni-node-name"
+        values   = ["*"]
+      }
+    }
+  }
+
+  dynamic "statement" {
+    for_each = var.enable_node_custom_tags_permissions ? [1] : []
+
+    content {
+      sid = "LoadBalancer"
+      actions = [
+        "elasticloadbalancing:CreateLoadBalancer",
+        "elasticloadbalancing:CreateTargetGroup",
+        "elasticloadbalancing:CreateListener",
+        "elasticloadbalancing:CreateRule",
+        "ec2:CreateSecurityGroup",
+      ]
+      resources = ["*"]
+
+      condition {
+        test     = "StringEquals"
+        variable = "aws:RequestTag/eks:eks-cluster-name"
+        values   = ["$${aws:PrincipalTag/eks:eks-cluster-name}"]
+      }
+    }
+  }
+
+  dynamic "statement" {
+    for_each = var.enable_node_custom_tags_permissions ? [1] : []
+
+    content {
+      sid       = "ShieldProtection"
+      actions   = ["shield:CreateProtection"]
+      resources = ["*"]
+
+      condition {
+        test     = "StringEquals"
+        variable = "aws:RequestTag/eks:eks-cluster-name"
+        values   = ["$${aws:PrincipalTag/eks:eks-cluster-name}"]
+      }
+    }
+  }
+
+  dynamic "statement" {
+    for_each = var.enable_node_custom_tags_permissions ? [1] : []
+
+    content {
+      sid       = "ShieldTagResource"
+      actions   = ["shield:TagResource"]
+      resources = ["arn:${local.partition}:shield::*:protection/*"]
+
+      condition {
+        test     = "StringEquals"
+        variable = "aws:RequestTag/eks:eks-cluster-name"
+        values   = ["$${aws:PrincipalTag/eks:eks-cluster-name}"]
+      }
+    }
+  }
+}
+
+resource "aws_iam_policy" "eks_auto_custom" {
+  count = local.create_node_iam_role_custom_policy ? 1 : 0
+
+  name        = var.node_iam_role_use_name_prefix ? null : local.node_iam_role_name
+  name_prefix = var.node_iam_role_use_name_prefix ? "${local.node_iam_role_name}-" : null
+  path        = var.node_iam_role_path
+  description = var.node_iam_role_description
+
+  policy = data.aws_iam_policy_document.eks_auto_custom[0].json
+
+  tags = merge(var.tags, var.node_iam_role_tags)
+}

variables.tf

@@ -620,6 +620,18 @@ variable "node_iam_role_tags" {
   default     = {}
 }
 
+variable "enable_node_custom_tags_permissions" {
+  description = "Determines whether to enable permissions for custom tags for the EKS Auto node IAM role"
+  type        = bool
+  default     = true
+}
+
+variable "node_iam_role_policy_statements" {
+  description = "A list of IAM policy [statements](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/iam_policy_document#statement) - used for adding specific IAM permissions as needed"
+  type        = any
+  default     = []
+}
+
 ################################################################################
 # Fargate
 ################################################################################