Update EKS test-infra (#1203)

This commit simplifies the EKS infrastructure by moving the config map
and kubeconfig file generation into the EKS cluster module. It also
adds a test for a progressive apply scenario involving file creation.

Author: dak1n1

kubernetes/test-infra/eks/kubernetes-config/main.tf

@@ -1,21 +1,13 @@
-resource "kubernetes_config_map" "name" {
-  metadata {
-    name      = "aws-auth"
-    namespace = "kube-system"
-  }
-
-  data = {
-    mapRoles = join(
-      "\n",
-      formatlist(local.mapped_role_format, var.k8s_node_role_arn),
-    )
-  }
-}
-
-# Optional: this kubeconfig file is only used for manual CLI access to the cluster.
-resource "null_resource" "generate-kubeconfig" {
-  provisioner "local-exec" {
-    command = "aws eks update-kubeconfig --name ${var.cluster_name} --kubeconfig ${path.root}/kubeconfig"
+terraform {
+  required_providers {
+    kubernetes = {
+      source = "localhost/test/kubernetes"
+      version = "9.9.9"
+    }
+    helm = {
+      source  = "localhost/test/helm"
+      version = "9.9.9"
+    }
   }
 }
 

kubernetes/test-infra/eks/main.tf

@@ -1,9 +1,19 @@
 terraform {
   required_providers {
-    kubernetes = {
+    # This is the locally compiled version of the provider, based on the current branch.
+    kubernetes-local = {
       source = "localhost/test/kubernetes"
       version = "9.9.9"
     }
+    # The following block configures the latest released version of the provider, which is needed for the EKS cluster module.
+    # This configuration is a work-around, because required_providers blocks are not inherited by sub-modules.
+    # A "required_providers" block needs to be added to all sub-modules in order to use a custom "source" and "version".
+    # Otherwise, the sub-module will use defaults, which in our case means an empty provider config.
+    # https://github.com/hashicorp/terraform/issues/27361
+    kubernetes-released = {
+      source = "hashicorp/kubernetes"
+      version = ">= 2.0.2"
+    }
     helm = {
       source  = "localhost/test/helm"
       version = "9.9.9"
@@ -25,7 +35,7 @@ data "aws_eks_cluster" "default" {
 # on the system running terraform, either in $PATH as shown below, or in another location, which can be
 # specified in the `command`.
 # See the commented provider blocks below for alternative configuration options.
-provider "kubernetes" {
+provider "kubernetes-released" {
   host                   = data.aws_eks_cluster.default.endpoint
   cluster_ca_certificate = base64decode(data.aws_eks_cluster.default.certificate_authority[0].data)
   exec {
@@ -35,15 +45,12 @@ provider "kubernetes" {
   }
 }
 
-# This configuration is also valid, but the token may expire during long-running applies.
-# data "aws_eks_cluster_auth" "default" {
-#  name = module.cluster.cluster_id
-#}
-#provider "kubernetes" {
-#  host                   = data.aws_eks_cluster.default.endpoint
-#  cluster_ca_certificate = base64decode(data.aws_eks_cluster.default.certificate_authority[0].data)
-#  token                  = data.aws_eks_cluster_auth.default.token
-#}
+# This tests a progressive apply scenario where the kubeconfig is created in the same apply as Kubernetes resources.
+# It should alert us to issues like this one before they're released.
+# https://github.com/hashicorp/terraform-provider-kubernetes/issues/1142
+provider "kubernetes-local" {
+  config_path = module.cluster.kubeconfig_filename
+}
 
 provider "helm" {
   kubernetes {
@@ -58,35 +65,38 @@ provider "helm" {
 }
 
 provider "aws" {
-  region = var.region
 }
 
 module "vpc" {
   source = "./vpc"
 }
 
 module "cluster" {
+  providers         =  {kubernetes = kubernetes-released}
   source  = "terraform-aws-modules/eks/aws"
   version = "14.0.0"
 
   vpc_id  = module.vpc.vpc_id
   subnets = module.vpc.subnets
 
-  cluster_name    = module.vpc.cluster_name
-  cluster_version = var.kubernetes_version
-  manage_aws_auth = false # Managed in ./kubernetes-config/main.tf instead.
-  # This kubeconfig expires in 15 minutes, so we'll use an exec block instead.
-  # See ./kubernetes-config/main.tf provider block for details.
-  write_kubeconfig = false
+  cluster_name     = module.vpc.cluster_name
+  cluster_version  = var.kubernetes_version
+  manage_aws_auth  = true
+  write_kubeconfig = true
 
+  # See this file for more options
+  # https://github.com/terraform-aws-modules/terraform-aws-eks/blob/master/local.tf#L28
   workers_group_defaults = {
     root_volume_type = "gp2"
   }
+
   worker_groups = [
     {
-      instance_type        = var.workers_type
-      asg_desired_capacity = var.workers_count
+      name                 = module.vpc.cluster_name
+      instance_type        = "m4.large"
+      asg_min_size         = 1
       asg_max_size         = 4
+      asg_desired_capacity = 2
     },
   ]
 
@@ -96,6 +106,7 @@ module "cluster" {
 }
 
 module "kubernetes-config" {
+  providers         =  {kubernetes = kubernetes-local}
   cluster_name      = module.cluster.cluster_id # creates dependency on cluster creation
   source            = "./kubernetes-config"
   k8s_node_role_arn = module.cluster.worker_iam_role_arn

kubernetes/test-infra/eks/variables.tf

@@ -1,21 +1,4 @@
-#
-# Variables Configuration
-#
-variable "region" {
-  default = "us-west-1"
-  type    = string
-}
-
 variable "kubernetes_version" {
   type    = string
   default = "1.18"
 }
-
-variable "workers_count" {
-  default = 2
-}
-
-variable "workers_type" {
-  type    = string
-  default = "m4.large"
-}

kubernetes/test-infra/eks/vpc/main.tf

@@ -1,4 +1,12 @@
-#
+terraform {
+  required_providers {
+    aws = {
+      source  = "hashicorp/aws"
+      version = "3.22.0"
+    }
+  }
+}
+
 # VPC Resources
 #  * VPC
 #  * Subnets