Adding new mixed type of worker group with instance overrides and mixed instances policy (#371)

* Adding new mixed type of worker group with instance overrides and mixed instances policy

* moving all count and lifecycle rule parameters to top/bottom

* adding custom IAM parts

* updating doc with new options

* fixes for spot instances

Author: max-rocket-internet

CHANGELOG.md

@@ -13,9 +13,10 @@ project adheres to [Semantic Versioning](http://semver.org/).
 
 - Added support for custom service linked role for Auto Scaling group (by @voanhduy1512)
 - Added support for custom IAM roles for cluster and workers (by @erks)
-- Add cluster arn to outputs (by @alexsn)
+- Added cluster ARN to outputs (by @alexsn)
 - Added outputs for `workers_user_data` and `workers_default_ami_id` (by @max-rocket-internet)
 - Added doc about spot instances (by @max-rocket-internet)
+- Added new worker group option with a mixed instances policy (by @max-rocket-internet)
 
 ### Changed
 

README.md

@@ -148,6 +148,8 @@ MIT Licensed. See [LICENSE](https://github.com/terraform-aws-modules/terraform-a
 | worker\_create\_security\_group | Whether to create a security group for the workers or attach the workers to `worker_security_group_id`. | string | `"true"` | no |
 | worker\_group\_count | The number of maps contained within the worker_groups list. | string | `"1"` | no |
 | worker\_group\_launch\_template\_count | The number of maps contained within the worker_groups_launch_template list. | string | `"0"` | no |
+| worker\_group\_launch\_template\_mixed | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no |
+| worker\_group\_launch\_template\_mixed\_count | The number of maps contained within the worker_group_launch_template_mixed list. | string | `"0"` | no |
 | worker\_group\_tags | A map defining extra tags to be applied to the worker group ASG. | map | `{ "default": [] }` | no |
 | worker\_groups | A list of maps defining worker group configurations to be defined using AWS Launch Configurations. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no |
 | worker\_groups\_launch\_template | A list of maps defining worker group configurations to be defined using AWS Launch Templates. See workers_group_defaults for valid keys. | list | `[ { "name": "default" } ]` | no |

aws_auth.tf

@@ -1,10 +1,11 @@
 resource "local_file" "config_map_aws_auth" {
+  count    = "${var.write_aws_auth_config ? 1 : 0}"
   content  = "${data.template_file.config_map_aws_auth.rendered}"
   filename = "${var.config_output_path}config-map-aws-auth_${var.cluster_name}.yaml"
-  count    = "${var.write_aws_auth_config ? 1 : 0}"
 }
 
 resource "null_resource" "update_config_map_aws_auth" {
+  count      = "${var.manage_aws_auth ? 1 : 0}"
   depends_on = ["aws_eks_cluster.this"]
 
   provisioner "local-exec" {
@@ -28,8 +29,6 @@ EOS
     config_map_rendered      = "${data.template_file.config_map_aws_auth.rendered}"
     endpoint                 = "${aws_eks_cluster.this.endpoint}"
   }
-
-  count = "${var.manage_aws_auth ? 1 : 0}"
 }
 
 data "aws_caller_identity" "current" {}

cluster.tf

@@ -23,52 +23,52 @@ resource "aws_eks_cluster" "this" {
 }
 
 resource "aws_security_group" "cluster" {
+  count       = "${var.cluster_create_security_group ? 1 : 0}"
   name_prefix = "${var.cluster_name}"
   description = "EKS cluster security group."
   vpc_id      = "${var.vpc_id}"
   tags        = "${merge(var.tags, map("Name", "${var.cluster_name}-eks_cluster_sg"))}"
-  count       = "${var.cluster_create_security_group ? 1 : 0}"
 }
 
 resource "aws_security_group_rule" "cluster_egress_internet" {
+  count             = "${var.cluster_create_security_group ? 1 : 0}"
   description       = "Allow cluster egress access to the Internet."
   protocol          = "-1"
   security_group_id = "${aws_security_group.cluster.id}"
   cidr_blocks       = ["0.0.0.0/0"]
   from_port         = 0
   to_port           = 0
   type              = "egress"
-  count             = "${var.cluster_create_security_group ? 1 : 0}"
 }
 
 resource "aws_security_group_rule" "cluster_https_worker_ingress" {
+  count                    = "${var.cluster_create_security_group ? 1 : 0}"
   description              = "Allow pods to communicate with the EKS cluster API."
   protocol                 = "tcp"
   security_group_id        = "${aws_security_group.cluster.id}"
   source_security_group_id = "${local.worker_security_group_id}"
   from_port                = 443
   to_port                  = 443
   type                     = "ingress"
-  count                    = "${var.cluster_create_security_group ? 1 : 0}"
 }
 
 resource "aws_iam_role" "cluster" {
+  count                 = "${var.manage_cluster_iam_resources ? 1 : 0}"
   name_prefix           = "${var.cluster_name}"
   assume_role_policy    = "${data.aws_iam_policy_document.cluster_assume_role_policy.json}"
   permissions_boundary  = "${var.permissions_boundary}"
   path                  = "${var.iam_path}"
   force_detach_policies = true
-  count                 = "${var.manage_cluster_iam_resources ? 1 : 0}"
 }
 
 resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSClusterPolicy" {
+  count      = "${var.manage_cluster_iam_resources ? 1 : 0}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonEKSClusterPolicy"
   role       = "${aws_iam_role.cluster.name}"
-  count      = "${var.manage_cluster_iam_resources ? 1 : 0}"
 }
 
 resource "aws_iam_role_policy_attachment" "cluster_AmazonEKSServicePolicy" {
+  count      = "${var.manage_cluster_iam_resources ? 1 : 0}"
   policy_arn = "arn:aws:iam::aws:policy/AmazonEKSServicePolicy"
   role       = "${aws_iam_role.cluster.name}"
-  count      = "${var.manage_cluster_iam_resources ? 1 : 0}"
 }

data.tf

@@ -58,22 +58,22 @@ data "template_file" "kubeconfig" {
 }
 
 data "template_file" "aws_authenticator_env_variables" {
+  count = "${length(var.kubeconfig_aws_authenticator_env_variables)}"
+
   template = <<EOF
         - name: $${key}
           value: $${value}
 EOF
 
-  count = "${length(var.kubeconfig_aws_authenticator_env_variables)}"
-
   vars {
     value = "${element(values(var.kubeconfig_aws_authenticator_env_variables), count.index)}"
     key   = "${element(keys(var.kubeconfig_aws_authenticator_env_variables), count.index)}"
   }
 }
 
 data "template_file" "userdata" {
-  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
   count    = "${var.worker_group_count}"
+  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
 
   vars {
     cluster_name         = "${aws_eks_cluster.this.name}"
@@ -87,8 +87,23 @@ data "template_file" "userdata" {
 }
 
 data "template_file" "launch_template_userdata" {
-  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
   count    = "${var.worker_group_launch_template_count}"
+  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
+
+  vars {
+    cluster_name         = "${aws_eks_cluster.this.name}"
+    endpoint             = "${aws_eks_cluster.this.endpoint}"
+    cluster_auth_base64  = "${aws_eks_cluster.this.certificate_authority.0.data}"
+    pre_userdata         = "${lookup(var.worker_groups_launch_template[count.index], "pre_userdata", local.workers_group_defaults["pre_userdata"])}"
+    additional_userdata  = "${lookup(var.worker_groups_launch_template[count.index], "additional_userdata", local.workers_group_defaults["additional_userdata"])}"
+    bootstrap_extra_args = "${lookup(var.worker_groups_launch_template[count.index], "bootstrap_extra_args", local.workers_group_defaults["bootstrap_extra_args"])}"
+    kubelet_extra_args   = "${lookup(var.worker_groups_launch_template[count.index], "kubelet_extra_args", local.workers_group_defaults["kubelet_extra_args"])}"
+  }
+}
+
+data "template_file" "workers_launch_template_mixed" {
+  count    = "${var.worker_group_launch_template_mixed_count}"
+  template = "${file("${path.module}/templates/userdata.sh.tpl")}"
 
   vars {
     cluster_name         = "${aws_eks_cluster.this.name}"
@@ -102,16 +117,21 @@ data "template_file" "launch_template_userdata" {
 }
 
 data "aws_iam_role" "custom_cluster_iam_role" {
-  name  = "${var.cluster_iam_role_name}"
   count = "${var.manage_cluster_iam_resources ? 0 : 1}"
+  name  = "${var.cluster_iam_role_name}"
 }
 
 data "aws_iam_instance_profile" "custom_worker_group_iam_instance_profile" {
-  name  = "${lookup(var.worker_groups[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
   count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_count}"
+  name  = "${lookup(var.worker_groups[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
 }
 
 data "aws_iam_instance_profile" "custom_worker_group_launch_template_iam_instance_profile" {
-  name  = "${lookup(var.worker_groups_launch_template[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
   count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_launch_template_count}"
+  name  = "${lookup(var.worker_groups_launch_template[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
+}
+
+data "aws_iam_instance_profile" "custom_worker_group_launch_template_mixed_iam_instance_profile" {
+  count = "${var.manage_worker_iam_resources ? 0 : var.worker_group_launch_template_mixed_count}"
+  name  = "${lookup(var.worker_group_launch_template_mixed[count.index], "iam_instance_profile_name", local.workers_group_defaults["iam_instance_profile_name"])}"
 }

docs/spot-instances.md

@@ -1,101 +1,98 @@
 # Using spot instances
 
-Spot instances usually cost around 30-70% less than an on-demand instance. So using them for your EKS workloads can save a lot of money but requires some special considerations as they will be terminated with only 2 minutes warning.
+Spot instances usually cost around 30-70% less than an on-demand instance. So using them for your EKS workloads can save a lot of money but requires some special considerations as they could be terminated with only 2 minutes warning.
 
 You need to install a daemonset to catch the 2 minute warning before termination. This will ensure the node is gracefully drained before termination. You can install the [k8s-spot-termination-handler](https://github.com/kube-aws/kube-spot-termination-notice-handler) for this. There's a [Helm chart](https://github.com/helm/charts/tree/master/stable/k8s-spot-termination-handler):
 
 ```
 helm install stable/k8s-spot-termination-handler --namespace kube-system
 ```
 
-In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler):
+In the following examples at least 1 worker group that uses on-demand instances is included. This worker group has an added node label that can be used in scheduling. This could be used to schedule any workload not suitable for spot instances but is important for the [cluster-autoscaler](https://github.com/kubernetes/autoscaler/tree/master/cluster-autoscaler) as it might be end up unscheduled when spot instances are terminated. You can add this to the values of the [cluster-autoscaler helm chart](https://github.com/helm/charts/tree/master/stable/cluster-autoscaler):
 
 ```yaml
 nodeSelector:
-  spot: "false"
+  kubernetes.io/lifecycle: spot
 ```
 
 Notes:
 
 - The `spot_price` is set to the on-demand price so that the spot instances will run as long as they are the cheaper.
 - It's best to have a broad range of instance types to ensure there's always some instances to run when prices fluctuate.
-- Using an [AWS Spot Fleet](https://docs.aws.amazon.com/AWSEC2/latest/UserGuide/spot-fleet-requests.html) is the best option but is not supported by this module yet.
 - There is an AWS blog article about this [here](https://aws.amazon.com/blogs/compute/run-your-kubernetes-workloads-on-amazon-ec2-spot-instances-with-amazon-eks/).
 - Consider using [k8s-spot-rescheduler](https://github.com/pusher/k8s-spot-rescheduler) to move pods from on-demand to spot instances.
 
 ## Using Launch Configuration
 
-Example Terraform worker group configuration that use an ASG with launch configuration:
+Example worker group configuration that uses an ASG with launch configuration for each worker group:
 
 ```hcl
-worker_group_count = 3
-
-worker_groups = [
-  {
-    name                = "on-demand-1"
-    instance_type       = "m4.xlarge"
-    asg_max_size        = 1
-    autoscaling_enabled = true
-    kubelet_extra_args  = "--node-labels=spot=false"
-    suspended_processes = "AZRebalance"
-  },
-  {
-    name                = "spot-1"
-    spot_price          = "0.39"
-    instance_type       = "c4.2xlarge"
-    asg_max_size        = 20
-    autoscaling_enabled = true
-    kubelet_extra_args  = "--node-labels=spot=true"
-    suspended_processes = "AZRebalance"
-  },
-  {
-    name                = "spot-2"
-    spot_price          = "0.40"
-    instance_type       = "m4.2xlarge"
-    asg_max_size        = 20
-    autoscaling_enabled = true
-    kubelet_extra_args  = "--node-labels=spot=true"
-    suspended_processes = "AZRebalance"
-  }
-]
+  worker_group_count = 3
+
+  worker_groups = [
+    {
+      name                = "on-demand-1"
+      instance_type       = "m4.xlarge"
+      asg_max_size        = 1
+      autoscaling_enabled = true
+      kubelet_extra_args  = "--node-labels=kubernetes.io/lifecycle=normal"
+      suspended_processes = "AZRebalance"
+    },
+    {
+      name                = "spot-1"
+      spot_price          = "0.199"
+      instance_type       = "c4.xlarge"
+      asg_max_size        = 20
+      autoscaling_enabled = true
+      kubelet_extra_args  = "--node-labels=kubernetes.io/lifecycle=spot"
+      suspended_processes = "AZRebalance"
+    },
+    {
+      name                = "spot-2"
+      spot_price          = "0.20"
+      instance_type       = "m4.xlarge"
+      asg_max_size        = 20
+      autoscaling_enabled = true
+      kubelet_extra_args  = "--node-labels=kubernetes.io/lifecycle=spot"
+      suspended_processes = "AZRebalance"
+    }
+  ]
 ```
 
 ## Using Launch Templates
 
-Launch Template support is a recent addition to both AWS and this module. It might not be as tried and tested.
-
-Example Terraform worker group configuration that use an ASG with a launch template:
+Launch Template support is a recent addition to both AWS and this module. It might not be as tried and tested but it's more suitable for spot instances as it allowed multiple instance types in the same worker group:
 
 ```hcl
-
-worker_group_count = 1
-
-worker_groups = [
-  {
-    name                = "on-demand-1"
-    instance_type       = "m4.xlarge"
-    asg_max_size        = 10
-    autoscaling_enabled = true
-    kubelet_extra_args  = "--node-labels=spot=false"
-    suspended_processes = "AZRebalance"
-  }
-]
-
-worker_group_launch_template_count = 1
-
-worker_groups_launch_template = [
-  {
-    name                                     = "spot-1"
-    instance_type                            = "m5.xlarge"
-    override_instance_type                   = "m4.xlarge"
-    spot_instance_pools                      = 2
-    on_demand_percentage_above_base_capacity = 0
-    spot_max_price                           = "0.384"
-    asg_max_size                             = 10
-    autoscaling_enabled                      = true
-    kubelet_extra_args                       = "--node-labels=spot=true"
-  }
-]
+  worker_group_count = 1
+
+  worker_groups = [
+    {
+      name                = "on-demand-1"
+      instance_type       = "m4.xlarge"
+      asg_max_size        = 10
+      autoscaling_enabled = true
+      kubelet_extra_args  = "--node-labels=spot=false"
+      suspended_processes = "AZRebalance"
+    }
+  ]
+
+  worker_group_launch_template_mixed_count = 1
+
+  worker_group_launch_template_mixed = [
+    {
+      name                     = "spot-1"
+      override_instance_type_1 = "m5.large"
+      override_instance_type_2 = "c5.large"
+      override_instance_type_3 = "t3.large"
+      override_instance_type_4 = "r5.large"
+      spot_instance_pools      = 3
+      asg_max_size             = 5
+      asg_desired_size         = 5
+      autoscaling_enabled      = true
+      kubelet_extra_args       = "--node-labels=kubernetes.io/lifecycle=spot"
+    }
+  ]
 ```
 
 ## Important issues

kubectl.tf

@@ -1,5 +1,5 @@
 resource "local_file" "kubeconfig" {
+  count    = "${var.write_kubeconfig ? 1 : 0}"
   content  = "${data.template_file.kubeconfig.rendered}"
   filename = "${var.config_output_path}kubeconfig_${var.cluster_name}"
-  count    = "${var.write_kubeconfig ? 1 : 0}"
 }

local.tf

@@ -53,6 +53,18 @@ locals {
     launch_template_placement_group   = ""                                            # The name of the placement group into which to launch the instances, if any.
     root_encrypted                    = ""                                            # Whether the volume should be encrypted or not
     eni_delete                        = true                                          # Delete the ENI on termination (if set to false you will have to manually delete before destroying)
+
+    # Settings for launch templates with mixed instances policy
+    override_instance_type_1                 = "m5.large"     # Override instance type 1 for mixed instances policy
+    override_instance_type_2                 = "c5.large"     # Override instance type 2 for mixed instances policy
+    override_instance_type_3                 = "t3.large"     # Override instance type 3 for mixed instances policy
+    override_instance_type_4                 = "r5.large"     # Override instance type 4 for mixed instances policy
+    on_demand_allocation_strategy            = "prioritized"  # Strategy to use when launching on-demand instances. Valid values: prioritized.
+    on_demand_base_capacity                  = "0"            # Absolute minimum amount of desired capacity that must be fulfilled by on-demand instances
+    on_demand_percentage_above_base_capacity = "0"            # Percentage split between on-demand and Spot instances above the base on-demand capacity
+    spot_allocation_strategy                 = "lowest-price" # The only valid value is lowest-price, which is also the default value. The Auto Scaling group selects the cheapest Spot pools and evenly allocates your Spot capacity across the number of Spot pools that you specify.
+    spot_instance_pools                      = 10             # "Number of Spot pools per availability zone to allocate capacity. EC2 Auto Scaling selects the cheapest Spot pools and evenly allocates Spot capacity across the number of Spot pools that you specify."
+    spot_max_price                           = ""             # Maximum price per unit hour that the user is willing to pay for the Spot instances. Default is the on-demand price
   }
 
   workers_group_defaults = "${merge(local.workers_group_defaults_defaults, var.workers_group_defaults)}"