feat: Improve addon dependency chain and decrease time to provision addons (due to retries) (#3218)

* feat: Improve addon dependency chain and decrease time to provision addons (due to retries)

* fix: Run pre-commit to clean up docs

Author: bryantbiggs

main.tf

@@ -208,7 +208,7 @@ locals {
 resource "aws_eks_access_entry" "this" {
   for_each = { for k, v in local.merged_access_entries : k => v if local.create }
 
-  cluster_name      = aws_eks_cluster.this[0].name
+  cluster_name      = aws_eks_cluster.this[0].id
   kubernetes_groups = try(each.value.kubernetes_groups, null)
   principal_arn     = each.value.principal_arn
   type              = try(each.value.type, "STANDARD")
@@ -225,7 +225,7 @@ resource "aws_eks_access_policy_association" "this" {
     type       = each.value.association_access_scope_type
   }
 
-  cluster_name = aws_eks_cluster.this[0].name
+  cluster_name = aws_eks_cluster.this[0].id
 
   policy_arn    = each.value.association_policy_arn
   principal_arn = each.value.principal_arn
@@ -481,19 +481,25 @@ resource "aws_iam_policy" "cluster_encryption" {
 # EKS Addons
 ################################################################################
 
+locals {
+  # TODO - Set to `NONE` on next breaking change when default addons are disabled
+  resolve_conflicts_on_create_default = var.bootstrap_self_managed_addons ? "OVERWRITE" : "NONE"
+}
+
 data "aws_eks_addon_version" "this" {
   for_each = { for k, v in var.cluster_addons : k => v if local.create && !local.create_outposts_local_cluster }
 
   addon_name         = try(each.value.name, each.key)
   kubernetes_version = coalesce(var.cluster_version, aws_eks_cluster.this[0].version)
-  most_recent        = try(each.value.most_recent, null)
+  # TODO - Set default fallback to  `true` on next breaking change
+  most_recent = try(each.value.most_recent, null)
 }
 
 resource "aws_eks_addon" "this" {
   # Not supported on outposts
   for_each = { for k, v in var.cluster_addons : k => v if !try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster }
 
-  cluster_name = aws_eks_cluster.this[0].name
+  cluster_name = aws_eks_cluster.this[0].id
   addon_name   = try(each.value.name, each.key)
 
   addon_version        = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version)
@@ -508,8 +514,9 @@ resource "aws_eks_addon" "this" {
     }
   }
 
-  preserve                    = try(each.value.preserve, true)
-  resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, "OVERWRITE")
+  preserve = try(each.value.preserve, true)
+  # TODO - Set to `NONE` on next breaking change when default addons are disabled
+  resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, local.resolve_conflicts_on_create_default)
   resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE")
   service_account_role_arn    = try(each.value.service_account_role_arn, null)
 
@@ -532,7 +539,7 @@ resource "aws_eks_addon" "before_compute" {
   # Not supported on outposts
   for_each = { for k, v in var.cluster_addons : k => v if try(v.before_compute, false) && local.create && !local.create_outposts_local_cluster }
 
-  cluster_name = aws_eks_cluster.this[0].name
+  cluster_name = aws_eks_cluster.this[0].id
   addon_name   = try(each.value.name, each.key)
 
   addon_version        = coalesce(try(each.value.addon_version, null), data.aws_eks_addon_version.this[each.key].version)
@@ -547,8 +554,9 @@ resource "aws_eks_addon" "before_compute" {
     }
   }
 
-  preserve                    = try(each.value.preserve, true)
-  resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, "OVERWRITE")
+  preserve = try(each.value.preserve, true)
+  # TODO - Set to `NONE` on next breaking change when default addons are disabled
+  resolve_conflicts_on_create = try(each.value.resolve_conflicts_on_create, local.resolve_conflicts_on_create_default)
   resolve_conflicts_on_update = try(each.value.resolve_conflicts_on_update, "OVERWRITE")
   service_account_role_arn    = try(each.value.service_account_role_arn, null)
 
@@ -570,14 +578,15 @@ locals {
   # Maintain current behavior for <= 1.29, remove default for >= 1.30
   # `null` will return the latest Kubernetes version from the EKS API, which at time of writing is 1.30
   # https://github.com/kubernetes/kubernetes/pull/123561
+  # TODO - remove on next breaking change in conjunction with issuer URL change below
   idpc_backwards_compat_version = contains(["1.21", "1.22", "1.23", "1.24", "1.25", "1.26", "1.27", "1.28", "1.29"], coalesce(var.cluster_version, "1.30"))
   idpc_issuer_url               = local.idpc_backwards_compat_version ? try(aws_eks_cluster.this[0].identity[0].oidc[0].issuer, null) : null
 }
 
 resource "aws_eks_identity_provider_config" "this" {
   for_each = { for k, v in var.cluster_identity_providers : k => v if local.create && !local.create_outposts_local_cluster }
 
-  cluster_name = aws_eks_cluster.this[0].name
+  cluster_name = aws_eks_cluster.this[0].id
 
   oidc {
     client_id                     = each.value.client_id

node_groups.tf

@@ -32,7 +32,7 @@ resource "time_sleep" "this" {
   create_duration = var.dataplane_wait_duration
 
   triggers = {
-    cluster_name         = aws_eks_cluster.this[0].name
+    cluster_name         = aws_eks_cluster.this[0].id
     cluster_endpoint     = aws_eks_cluster.this[0].endpoint
     cluster_version      = aws_eks_cluster.this[0].version
     cluster_service_cidr = var.cluster_ip_family == "ipv6" ? try(local.kubernetes_network_config.service_ipv6_cidr, "") : try(local.kubernetes_network_config.service_ipv4_cidr, "")

tests/fast-addons/README.md

@@ -0,0 +1,92 @@
+# Fast Addons
+
+Refer to https://github.com/terraform-aws-modules/terraform-aws-eks/pull/3214 for additional information.
+
+<!-- TODO - remove this at next breaking change since the defaults will be in place -->
+
+## Usage
+
+To provision the provided configurations you need to execute:
+
+```bash
+$ terraform init
+$ terraform plan
+$ terraform apply --auto-approve
+```
+
+Note that this example may create resources which cost money. Run `terraform destroy` when you don't need these resources.
+
+<!-- BEGIN_TF_DOCS -->
+## Requirements
+
+| Name | Version |
+|------|---------|
+| <a name="requirement_terraform"></a> [terraform](#requirement\_terraform) | >= 1.3.2 |
+| <a name="requirement_aws"></a> [aws](#requirement\_aws) | >= 5.75 |
+
+## Providers
+
+| Name | Version |
+|------|---------|
+| <a name="provider_aws"></a> [aws](#provider\_aws) | >= 5.75 |
+
+## Modules
+
+| Name | Source | Version |
+|------|--------|---------|
+| <a name="module_eks"></a> [eks](#module\_eks) | ../.. | n/a |
+| <a name="module_vpc"></a> [vpc](#module\_vpc) | terraform-aws-modules/vpc/aws | ~> 5.0 |
+
+## Resources
+
+| Name | Type |
+|------|------|
+| [aws_route_table_association.custom_network](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/route_table_association) | resource |
+| [aws_subnet.custom_network](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/subnet) | resource |
+| [aws_vpc_ipv4_cidr_block_association.custom_network](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/resources/vpc_ipv4_cidr_block_association) | resource |
+| [aws_availability_zones.available](https://registry.terraform.io/providers/hashicorp/aws/latest/docs/data-sources/availability_zones) | data source |
+
+## Inputs
+
+No inputs.
+
+## Outputs
+
+| Name | Description |
+|------|-------------|
+| <a name="output_access_entries"></a> [access\_entries](#output\_access\_entries) | Map of access entries created and their attributes |
+| <a name="output_cloudwatch_log_group_arn"></a> [cloudwatch\_log\_group\_arn](#output\_cloudwatch\_log\_group\_arn) | Arn of cloudwatch log group created |
+| <a name="output_cloudwatch_log_group_name"></a> [cloudwatch\_log\_group\_name](#output\_cloudwatch\_log\_group\_name) | Name of cloudwatch log group created |
+| <a name="output_cluster_addons"></a> [cluster\_addons](#output\_cluster\_addons) | Map of attribute maps for all EKS cluster addons enabled |
+| <a name="output_cluster_arn"></a> [cluster\_arn](#output\_cluster\_arn) | The Amazon Resource Name (ARN) of the cluster |
+| <a name="output_cluster_certificate_authority_data"></a> [cluster\_certificate\_authority\_data](#output\_cluster\_certificate\_authority\_data) | Base64 encoded certificate data required to communicate with the cluster |
+| <a name="output_cluster_dualstack_oidc_issuer_url"></a> [cluster\_dualstack\_oidc\_issuer\_url](#output\_cluster\_dualstack\_oidc\_issuer\_url) | Dual-stack compatible URL on the EKS cluster for the OpenID Connect identity provider |
+| <a name="output_cluster_endpoint"></a> [cluster\_endpoint](#output\_cluster\_endpoint) | Endpoint for your Kubernetes API server |
+| <a name="output_cluster_iam_role_arn"></a> [cluster\_iam\_role\_arn](#output\_cluster\_iam\_role\_arn) | IAM role ARN of the EKS cluster |
+| <a name="output_cluster_iam_role_name"></a> [cluster\_iam\_role\_name](#output\_cluster\_iam\_role\_name) | IAM role name of the EKS cluster |
+| <a name="output_cluster_iam_role_unique_id"></a> [cluster\_iam\_role\_unique\_id](#output\_cluster\_iam\_role\_unique\_id) | Stable and unique string identifying the IAM role |
+| <a name="output_cluster_id"></a> [cluster\_id](#output\_cluster\_id) | The ID of the EKS cluster. Note: currently a value is returned only for local EKS clusters created on Outposts |
+| <a name="output_cluster_identity_providers"></a> [cluster\_identity\_providers](#output\_cluster\_identity\_providers) | Map of attribute maps for all EKS identity providers enabled |
+| <a name="output_cluster_ip_family"></a> [cluster\_ip\_family](#output\_cluster\_ip\_family) | The IP family used by the cluster (e.g. `ipv4` or `ipv6`) |
+| <a name="output_cluster_name"></a> [cluster\_name](#output\_cluster\_name) | The name of the EKS cluster |
+| <a name="output_cluster_oidc_issuer_url"></a> [cluster\_oidc\_issuer\_url](#output\_cluster\_oidc\_issuer\_url) | The URL on the EKS cluster for the OpenID Connect identity provider |
+| <a name="output_cluster_platform_version"></a> [cluster\_platform\_version](#output\_cluster\_platform\_version) | Platform version for the cluster |
+| <a name="output_cluster_primary_security_group_id"></a> [cluster\_primary\_security\_group\_id](#output\_cluster\_primary\_security\_group\_id) | Cluster security group that was created by Amazon EKS for the cluster. Managed node groups use this security group for control-plane-to-data-plane communication. Referred to as 'Cluster security group' in the EKS console |
+| <a name="output_cluster_security_group_arn"></a> [cluster\_security\_group\_arn](#output\_cluster\_security\_group\_arn) | Amazon Resource Name (ARN) of the cluster security group |
+| <a name="output_cluster_security_group_id"></a> [cluster\_security\_group\_id](#output\_cluster\_security\_group\_id) | ID of the cluster security group |
+| <a name="output_cluster_service_cidr"></a> [cluster\_service\_cidr](#output\_cluster\_service\_cidr) | The CIDR block where Kubernetes pod and service IP addresses are assigned from |
+| <a name="output_cluster_status"></a> [cluster\_status](#output\_cluster\_status) | Status of the EKS cluster. One of `CREATING`, `ACTIVE`, `DELETING`, `FAILED` |
+| <a name="output_cluster_tls_certificate_sha1_fingerprint"></a> [cluster\_tls\_certificate\_sha1\_fingerprint](#output\_cluster\_tls\_certificate\_sha1\_fingerprint) | The SHA1 fingerprint of the public key of the cluster's certificate |
+| <a name="output_eks_managed_node_groups"></a> [eks\_managed\_node\_groups](#output\_eks\_managed\_node\_groups) | Map of attribute maps for all EKS managed node groups created |
+| <a name="output_eks_managed_node_groups_autoscaling_group_names"></a> [eks\_managed\_node\_groups\_autoscaling\_group\_names](#output\_eks\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by EKS managed node groups |
+| <a name="output_fargate_profiles"></a> [fargate\_profiles](#output\_fargate\_profiles) | Map of attribute maps for all EKS Fargate Profiles created |
+| <a name="output_kms_key_arn"></a> [kms\_key\_arn](#output\_kms\_key\_arn) | The Amazon Resource Name (ARN) of the key |
+| <a name="output_kms_key_id"></a> [kms\_key\_id](#output\_kms\_key\_id) | The globally unique identifier for the key |
+| <a name="output_kms_key_policy"></a> [kms\_key\_policy](#output\_kms\_key\_policy) | The IAM resource policy set on the key |
+| <a name="output_node_security_group_arn"></a> [node\_security\_group\_arn](#output\_node\_security\_group\_arn) | Amazon Resource Name (ARN) of the node shared security group |
+| <a name="output_node_security_group_id"></a> [node\_security\_group\_id](#output\_node\_security\_group\_id) | ID of the node shared security group |
+| <a name="output_oidc_provider"></a> [oidc\_provider](#output\_oidc\_provider) | The OpenID Connect identity provider (issuer URL without leading `https://`) |
+| <a name="output_oidc_provider_arn"></a> [oidc\_provider\_arn](#output\_oidc\_provider\_arn) | The ARN of the OIDC Provider if `enable_irsa = true` |
+| <a name="output_self_managed_node_groups"></a> [self\_managed\_node\_groups](#output\_self\_managed\_node\_groups) | Map of attribute maps for all self managed node groups created |
+| <a name="output_self_managed_node_groups_autoscaling_group_names"></a> [self\_managed\_node\_groups\_autoscaling\_group\_names](#output\_self\_managed\_node\_groups\_autoscaling\_group\_names) | List of the autoscaling group names created by self-managed node groups |
+<!-- END_TF_DOCS -->

tests/fast-addons/main.tf

@@ -0,0 +1,159 @@
+provider "aws" {
+  region = local.region
+}
+
+locals {
+  name            = "ex-${basename(path.cwd)}"
+  cluster_version = "1.31"
+  region          = "eu-west-1"
+
+  tags = {
+    Test       = local.name
+    GithubRepo = "terraform-aws-eks"
+    GithubOrg  = "terraform-aws-modules"
+  }
+}
+
+################################################################################
+# EKS Module
+################################################################################
+
+module "eks" {
+  source = "../.."
+
+  cluster_name                   = local.name
+  cluster_version                = local.cluster_version
+  cluster_endpoint_public_access = true
+
+  enable_cluster_creator_admin_permissions = true
+
+  # Disable the default self-managed addons to avoid the penalty of adopting them later
+  bootstrap_self_managed_addons = false
+
+  # Addons will be provisioned net new via the EKS addon API
+  cluster_addons = {
+    coredns = {
+      most_recent = true
+    }
+    eks-pod-identity-agent = {
+      before_compute = true
+      most_recent    = true
+    }
+    kube-proxy = {
+      most_recent = true
+    }
+    vpc-cni = {
+      most_recent    = true
+      before_compute = true
+      configuration_values = jsonencode({
+        env = {
+          # Use subnet tags to avoid the need to inject the ENIConfig
+          # which requires a live API server endpoint which leads to a dependency of:
+          # Control plane -> API request to create ENIConfig -> VPC CNI addon -> nodes/compute
+          # With the subnet discovery feature, we can avoid this dependency:
+          # Control plane -> VPC CNI addon -> nodes/compute
+          ENABLE_SUBNET_DISCOVERY = "true"
+        }
+      })
+    }
+  }
+
+  vpc_id     = module.vpc.vpc_id
+  subnet_ids = module.vpc.private_subnets
+
+  eks_managed_node_groups = {
+    example = {
+      instance_types = ["m6i.large"]
+
+      min_size     = 2
+      max_size     = 5
+      desired_size = 2
+    }
+  }
+
+  tags = local.tags
+}
+
+################################################################################
+# VPC
+################################################################################
+
+data "aws_availability_zones" "available" {
+  # Exclude local zones
+  filter {
+    name   = "opt-in-status"
+    values = ["opt-in-not-required"]
+  }
+}
+
+locals {
+  vpc_cidr = "10.0.0.0/16"
+  azs      = slice(data.aws_availability_zones.available.names, 0, 3)
+}
+
+module "vpc" {
+  source  = "terraform-aws-modules/vpc/aws"
+  version = "~> 5.0"
+
+  name = local.name
+  cidr = local.vpc_cidr
+
+  azs             = local.azs
+  private_subnets = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 4, k)]
+  public_subnets  = [for k, v in local.azs : cidrsubnet(local.vpc_cidr, 8, k + 48)]
+
+  enable_nat_gateway = true
+  single_nat_gateway = true
+
+  public_subnet_tags = {
+    "kubernetes.io/role/elb" = 1
+  }
+
+  tags = local.tags
+}
+
+################################################################################
+# Custom Networking
+################################################################################
+
+locals {
+  custom_network_vpc_cidr = "10.99.0.0/16"
+
+  custom_network_subnets = [for k, v in local.azs : cidrsubnet(local.custom_network_vpc_cidr, 4, k)]
+}
+
+resource "aws_vpc_ipv4_cidr_block_association" "custom_network" {
+  vpc_id     = module.vpc.vpc_id
+  cidr_block = local.custom_network_vpc_cidr
+}
+
+resource "aws_subnet" "custom_network" {
+  count = length(local.custom_network_subnets)
+
+  vpc_id     = module.vpc.vpc_id
+  cidr_block = element(local.custom_network_subnets, count.index)
+
+  tags = merge(
+    local.tags,
+    {
+      # Tag for subnet discovery
+      "kubernetes.io/role/cni"          = 1
+      "kubernetes.io/role/internal-elb" = 1
+    }
+  )
+
+  depends_on = [
+    aws_vpc_ipv4_cidr_block_association.custom_network
+  ]
+}
+
+resource "aws_route_table_association" "custom_network" {
+  count = length(local.custom_network_subnets)
+
+  subnet_id      = element(aws_subnet.custom_network[*].id, count.index)
+  route_table_id = element(module.vpc.private_route_table_ids, 0)
+
+  depends_on = [
+    aws_vpc_ipv4_cidr_block_association.custom_network
+  ]
+}