Fix for remote access and lifecycle issues (#30)

Author: woz5999

README.md

@@ -208,7 +208,7 @@ Available targets:
 | delimiter | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes`.<br>Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no |
 | desired\_size | Initial desired number of worker nodes (external changes ignored) | `number` | n/a | yes |
 | disk\_size | Disk size in GiB for worker nodes. Defaults to 20. Ignored it `launch_template_id` is supplied.<br>Terraform will only perform drift detection if a configuration value is provided. | `number` | `20` | no |
-| ec2\_ssh\_key | SSH key name that should be used to access the worker nodes | `string` | `null` | no |
+| ec2\_ssh\_key | SSH key pair name to use to access the worker nodes | `string` | `null` | no |
 | enable\_cluster\_autoscaler | Set true to allow Kubernetes Cluster Auto Scaler to scale the node group | `bool` | `false` | no |
 | enabled | Set to false to prevent the module from creating any resources | `bool` | `null` | no |
 | environment | Environment, e.g. 'uw2', 'us-west-2', OR 'prod', 'staging', 'dev', 'UAT' | `string` | `null` | no |
@@ -230,18 +230,19 @@ Available targets:
 | namespace | Namespace, which could be your organization name or abbreviation, e.g. 'eg' or 'cp' | `string` | `null` | no |
 | regex\_replace\_chars | Regex to replace chars with empty string in `namespace`, `environment`, `stage` and `name`.<br>If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. | `string` | `null` | no |
 | resources\_to\_tag | List of auto-launched resource types to tag. Valid types are "instance", "volume", "elastic-gpu", "spot-instances-request". | `list(string)` | `[]` | no |
-| source\_security\_group\_ids | Set of EC2 Security Group IDs to allow SSH access (port 22) from on the worker nodes. If you specify `ec2_ssh_key`, but do not specify this configuration when you create an EKS Node Group, port 22 on the worker nodes is opened to the Internet (0.0.0.0/0) | `list(string)` | `[]` | no |
+| source\_security\_group\_ids | Set of EC2 Security Group IDs to allow SSH access (port 22) to the worker nodes. If you specify `ec2_ssh_key`, but do not specify this configuration when you create an EKS Node Group, port 22 on the worker nodes is opened to the Internet (0.0.0.0/0) | `list(string)` | `[]` | no |
 | stage | Stage, e.g. 'prod', 'staging', 'dev', OR 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no |
 | subnet\_ids | A list of subnet IDs to launch resources in | `list(string)` | n/a | yes |
 | tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no |
-| userdata\_override | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module<br>may generate "user data" that expects to find that script. If you want to use an AMI that is not<br>compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override` to provide<br>your own (Base64 encoded) user data. Use "" to prevent any user data from being set.<br><br>Setting `userdata_override` disables `kubernetes_taints`, `kubelet_additional_options`,<br>`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `string` | `null` | no |
+| userdata\_override\_base64 | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module<br>may generate "user data" that expects to find that script. If you want to use an AMI that is not<br>compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override_base64` to provide<br>your own (Base64 encoded) user data. Use "" to prevent any user data from being set.<br><br>Setting `userdata_override_base64` disables `kubernetes_taints`, `kubelet_additional_options`,<br>`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `string` | `null` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
 | eks\_node\_group\_arn | Amazon Resource Name (ARN) of the EKS Node Group |
 | eks\_node\_group\_id | EKS Cluster name and EKS Node Group name separated by a colon |
+| eks\_node\_group\_remote\_access\_security\_group\_id | The ID of the security group generated to allow SSH access to the nodes, if this module generated one |
 | eks\_node\_group\_resources | List of objects containing information about underlying resources of the EKS Node Group |
 | eks\_node\_group\_role\_arn | ARN of the worker nodes IAM role |
 | eks\_node\_group\_role\_name | Name of the worker nodes IAM role |

docs/terraform.md

@@ -34,7 +34,7 @@
 | delimiter | Delimiter to be used between `namespace`, `environment`, `stage`, `name` and `attributes`.<br>Defaults to `-` (hyphen). Set to `""` to use no delimiter at all. | `string` | `null` | no |
 | desired\_size | Initial desired number of worker nodes (external changes ignored) | `number` | n/a | yes |
 | disk\_size | Disk size in GiB for worker nodes. Defaults to 20. Ignored it `launch_template_id` is supplied.<br>Terraform will only perform drift detection if a configuration value is provided. | `number` | `20` | no |
-| ec2\_ssh\_key | SSH key name that should be used to access the worker nodes | `string` | `null` | no |
+| ec2\_ssh\_key | SSH key pair name to use to access the worker nodes | `string` | `null` | no |
 | enable\_cluster\_autoscaler | Set true to allow Kubernetes Cluster Auto Scaler to scale the node group | `bool` | `false` | no |
 | enabled | Set to false to prevent the module from creating any resources | `bool` | `null` | no |
 | environment | Environment, e.g. 'uw2', 'us-west-2', OR 'prod', 'staging', 'dev', 'UAT' | `string` | `null` | no |
@@ -56,18 +56,19 @@
 | namespace | Namespace, which could be your organization name or abbreviation, e.g. 'eg' or 'cp' | `string` | `null` | no |
 | regex\_replace\_chars | Regex to replace chars with empty string in `namespace`, `environment`, `stage` and `name`.<br>If not set, `"/[^a-zA-Z0-9-]/"` is used to remove all characters other than hyphens, letters and digits. | `string` | `null` | no |
 | resources\_to\_tag | List of auto-launched resource types to tag. Valid types are "instance", "volume", "elastic-gpu", "spot-instances-request". | `list(string)` | `[]` | no |
-| source\_security\_group\_ids | Set of EC2 Security Group IDs to allow SSH access (port 22) from on the worker nodes. If you specify `ec2_ssh_key`, but do not specify this configuration when you create an EKS Node Group, port 22 on the worker nodes is opened to the Internet (0.0.0.0/0) | `list(string)` | `[]` | no |
+| source\_security\_group\_ids | Set of EC2 Security Group IDs to allow SSH access (port 22) to the worker nodes. If you specify `ec2_ssh_key`, but do not specify this configuration when you create an EKS Node Group, port 22 on the worker nodes is opened to the Internet (0.0.0.0/0) | `list(string)` | `[]` | no |
 | stage | Stage, e.g. 'prod', 'staging', 'dev', OR 'source', 'build', 'test', 'deploy', 'release' | `string` | `null` | no |
 | subnet\_ids | A list of subnet IDs to launch resources in | `list(string)` | n/a | yes |
 | tags | Additional tags (e.g. `map('BusinessUnit','XYZ')` | `map(string)` | `{}` | no |
-| userdata\_override | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module<br>may generate "user data" that expects to find that script. If you want to use an AMI that is not<br>compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override` to provide<br>your own (Base64 encoded) user data. Use "" to prevent any user data from being set.<br><br>Setting `userdata_override` disables `kubernetes_taints`, `kubelet_additional_options`,<br>`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `string` | `null` | no |
+| userdata\_override\_base64 | Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module<br>may generate "user data" that expects to find that script. If you want to use an AMI that is not<br>compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override_base64` to provide<br>your own (Base64 encoded) user data. Use "" to prevent any user data from being set.<br><br>Setting `userdata_override_base64` disables `kubernetes_taints`, `kubelet_additional_options`,<br>`before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`. | `string` | `null` | no |
 
 ## Outputs
 
 | Name | Description |
 |------|-------------|
 | eks\_node\_group\_arn | Amazon Resource Name (ARN) of the EKS Node Group |
 | eks\_node\_group\_id | EKS Cluster name and EKS Node Group name separated by a colon |
+| eks\_node\_group\_remote\_access\_security\_group\_id | The ID of the security group generated to allow SSH access to the nodes, if this module generated one |
 | eks\_node\_group\_resources | List of objects containing information about underlying resources of the EKS Node Group |
 | eks\_node\_group\_role\_arn | ARN of the worker nodes IAM role |
 | eks\_node\_group\_role\_name | Name of the worker nodes IAM role |

main.tf

@@ -27,6 +27,7 @@ locals {
   configured_launch_template_version = length(local.configured_launch_template_name) > 0 && length(compact([var.launch_template_version])) > 0 ? var.launch_template_version : ""
 
   configured_ami_image_id = var.ami_image_id == null ? "" : var.ami_image_id
+  have_ssh_key            = var.ec2_ssh_key != null && var.ec2_ssh_key != ""
 
   # See https://aws.amazon.com/blogs/containers/introducing-launch-template-and-custom-ami-support-in-amazon-eks-managed-node-groups/
   features_require_ami = local.enabled && local.need_bootstrap
@@ -46,6 +47,12 @@ locals {
 
   launch_template_ami = length(local.configured_ami_image_id) == 0 ? (local.features_require_ami ? data.aws_ami.selected[0].image_id : "") : local.configured_ami_image_id
 
+  need_remote_access_sg = local.enabled && local.have_ssh_key && local.generate_launch_template
+  launch_template_vpc_security_group_ids = (
+    local.need_remote_access_sg ?
+    concat(data.aws_eks_cluster.this[0].vpc_config[*].cluster_security_group_id, aws_security_group.remote_access.*.id) : null
+  )
+
   autoscaler_enabled_tags = {
     "k8s.io/cluster-autoscaler/${var.cluster_name}" = "owned"
     "k8s.io/cluster-autoscaler/enabled"             = "true"
@@ -68,16 +75,14 @@ locals {
 
   aws_policy_prefix = format("arn:%s:iam::aws:policy", join("", data.aws_partition.current.*.partition))
 
-  get_cluster_data = local.enabled ? (local.need_cluster_kubernetes_version || local.need_bootstrap) : false
+  get_cluster_data = local.enabled ? (local.need_cluster_kubernetes_version || local.need_bootstrap || local.need_remote_access_sg) : false
 }
 
 data "aws_eks_cluster" "this" {
   count = local.get_cluster_data ? 1 : 0
   name  = var.cluster_name
 }
 
-
-
 module "label" {
   source = "git::https://github.com/cloudposse/terraform-null-label.git?ref=tags/0.19.2"
 
@@ -190,6 +195,7 @@ resource "aws_launch_template" "default" {
 
   instance_type = var.instance_types[0]
   image_id      = local.launch_template_ami == "" ? null : local.launch_template_ami
+  key_name      = local.have_ssh_key ? var.ec2_ssh_key : null
 
   dynamic "tag_specifications" {
     for_each = var.resources_to_tag
@@ -211,8 +217,13 @@ resource "aws_launch_template" "default" {
     http_endpoint = "enabled"
   }
 
-  user_data = local.userdata
-  tags      = local.node_group_tags
+  vpc_security_group_ids = local.launch_template_vpc_security_group_ids
+  user_data              = local.userdata
+  tags                   = local.node_group_tags
+
+  lifecycle {
+    create_before_destroy = true
+  }
 }
 
 data "aws_launch_template" "this" {
@@ -239,11 +250,14 @@ resource "random_pet" "cbd" {
     source_security_group_ids = join(",", var.source_security_group_ids)
     subnet_ids                = join(",", var.subnet_ids)
 
-    launch_template_id  = local.launch_template_id
-    launch_template_ami = local.launch_template_ami
+    launch_template_id  = local.use_launch_template ? local.launch_template_id : null
+    launch_template_ami = local.use_launch_template ? local.launch_template_ami : null
   }
 
   depends_on = [var.module_depends_on]
+  lifecycle {
+    create_before_destroy = true
+  }
 }
 
 
@@ -268,7 +282,9 @@ locals {
       min_size     = var.min_size
     }
 
-    ec2_ssh_key               = var.ec2_ssh_key == null ? "" : var.ec2_ssh_key
+    # Configure remote access via Launch Template if we are using one
+    need_remote_access        = local.have_ssh_key && ! local.use_launch_template
+    ec2_ssh_key               = var.ec2_ssh_key
     source_security_group_ids = var.source_security_group_ids
   }
 }
@@ -317,7 +333,7 @@ resource "aws_eks_node_group" "default" {
   }
 
   dynamic "remote_access" {
-    for_each = length(local.ng.ec2_ssh_key) > 0 ? ["true"] : []
+    for_each = local.ng.need_remote_access ? ["true"] : []
     content {
       ec2_ssh_key               = local.ng.ec2_ssh_key
       source_security_group_ids = local.ng.source_security_group_ids
@@ -378,7 +394,7 @@ resource "aws_eks_node_group" "cbd" {
   }
 
   dynamic "remote_access" {
-    for_each = length(local.ng.ec2_ssh_key) > 0 ? ["true"] : []
+    for_each = local.ng.need_remote_access ? ["true"] : []
     content {
       ec2_ssh_key               = local.ng.ec2_ssh_key
       source_security_group_ids = local.ng.source_security_group_ids

outputs.tf

@@ -27,3 +27,8 @@ output "eks_node_group_status" {
   description = "Status of the EKS Node Group"
   value       = join("", aws_eks_node_group.default.*.status, aws_eks_node_group.cbd.*.status)
 }
+
+output "eks_node_group_remote_access_security_group_id" {
+  description = "The ID of the security group generated to allow SSH access to the nodes, if this module generated one"
+  value       = join("", aws_security_group.remote_access.*.id)
+}

sg.tf

@@ -0,0 +1,33 @@
+# https://docs.aws.amazon.com/eks/latest/APIReference/API_RemoteAccessConfig.html
+
+resource "aws_security_group" "remote_access" {
+  count       = local.need_remote_access_sg ? 1 : 0
+  name        = format("%v%v%v", module.label.id, module.label.delimiter, "remoteAccess")
+  description = "Allow SSH access to all nodes in the nodeGroup"
+  vpc_id      = data.aws_eks_cluster.this[0].vpc_config[0].vpc_id
+  tags        = module.label.tags
+}
+
+resource "aws_security_group_rule" "remote_access_public_ssh" {
+  count       = local.need_remote_access_sg && length(var.source_security_group_ids) == 0 ? 1 : 0
+  description = "Allow SSH access to nodes from anywhere"
+  type        = "ingress"
+  protocol    = "tcp"
+  from_port   = 22
+  to_port     = 22
+  cidr_blocks = ["0.0.0.0/0"]
+
+  security_group_id = join("", aws_security_group.remote_access.*.id)
+}
+
+resource "aws_security_group_rule" "remote_access_source_sgs_ssh" {
+  for_each    = local.need_remote_access_sg ? toset(var.source_security_group_ids) : []
+  description = "Allow SSH access to nodes from security group"
+  type        = "ingress"
+  protocol    = "tcp"
+  from_port   = 22
+  to_port     = 22
+
+  security_group_id        = aws_security_group.remote_access[0].id
+  source_security_group_id = each.value
+}

userdata.tf

@@ -45,8 +45,8 @@ locals {
     local.userdata_vars.after_cluster_joining_userdata]
   )) > 0 : false
 
-  # If var.userdata_override = "" then we explicitly set userdata to ""
-  need_userdata = local.enabled && var.userdata_override == null ? (length(local.userdata_vars.before_cluster_joining_userdata) > 0) || local.need_bootstrap : false
+  # If var.userdata_override_base64 = "" then we explicitly set userdata to ""
+  need_userdata = local.enabled && var.userdata_override_base64 == null ? (length(local.userdata_vars.before_cluster_joining_userdata) > 0) || local.need_bootstrap : false
 
-  userdata = local.need_userdata ? base64encode(templatefile("${path.module}/userdata.tpl", merge(local.userdata_vars, local.cluster_data))) : var.userdata_override
+  userdata = local.need_userdata ? base64encode(templatefile("${path.module}/userdata.tpl", merge(local.userdata_vars, local.cluster_data))) : var.userdata_override_base64
 }

variables.tf

@@ -21,10 +21,16 @@ variable "create_before_destroy" {
 
 variable "ec2_ssh_key" {
   type        = string
-  description = "SSH key name that should be used to access the worker nodes"
+  description = "SSH key pair name to use to access the worker nodes"
   default     = null
 }
 
+variable "source_security_group_ids" {
+  type        = list(string)
+  default     = []
+  description = "Set of EC2 Security Group IDs to allow SSH access (port 22) to the worker nodes. If you specify `ec2_ssh_key`, but do not specify this configuration when you create an EKS Node Group, port 22 on the worker nodes is opened to the Internet (0.0.0.0/0)"
+}
+
 variable "desired_size" {
   type        = number
   description = "Initial desired number of worker nodes (external changes ignored)"
@@ -158,12 +164,6 @@ variable "kubernetes_version" {
   }
 }
 
-variable "source_security_group_ids" {
-  type        = list(string)
-  default     = []
-  description = "Set of EC2 Security Group IDs to allow SSH access (port 22) from on the worker nodes. If you specify `ec2_ssh_key`, but do not specify this configuration when you create an EKS Node Group, port 22 on the worker nodes is opened to the Internet (0.0.0.0/0)"
-}
-
 variable "module_depends_on" {
   type        = any
   default     = null
@@ -213,16 +213,16 @@ variable "bootstrap_additional_options" {
   description = "Additional options to bootstrap.sh. DO NOT include `--kubelet-additional-args`, use `kubelet_additional_args` var instead."
 }
 
-variable "userdata_override" {
+variable "userdata_override_base64" {
   type        = string
   default     = null
   description = <<-EOT
     Many features of this module rely on the `bootstrap.sh` provided with Amazon Linux, and this module
     may generate "user data" that expects to find that script. If you want to use an AMI that is not
-    compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override` to provide
+    compatible with the Amazon Linux `bootstrap.sh` initialization, then use `userdata_override_base64` to provide
     your own (Base64 encoded) user data. Use "" to prevent any user data from being set.
 
-    Setting `userdata_override` disables `kubernetes_taints`, `kubelet_additional_options`,
+    Setting `userdata_override_base64` disables `kubernetes_taints`, `kubelet_additional_options`,
     `before_cluster_joining_userdata`, `after_cluster_joining_userdata`, and `bootstrap_additional_options`.
     EOT
 }