fix: move EKS to auto mode

Author: bendehaan

aws/irsa.tf

@@ -72,6 +72,7 @@ resource "aws_iam_role" "user_role" {
   name = "cant-read-secrets"
 
   assume_role_policy = data.aws_iam_policy_document.user_assume_role.json
+  tags               = var.tags
 }
 
 data "aws_iam_policy_document" "user_assume_role" {
@@ -94,6 +95,7 @@ resource "aws_iam_policy" "secret_deny" {
   name_prefix = "secret-deny"
   description = "Deny secrets manager and SSM"
   policy      = data.aws_iam_policy_document.user_policy.json
+  tags        = var.tags
 }
 
 data "aws_iam_policy_document" "user_policy" {
@@ -123,15 +125,16 @@ data "aws_iam_policy_document" "user_policy" {
 }
 
 module "ebs_csi_irsa_role" {
-  source                = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  source                = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts"
   version               = "~> 6.0"
-  role_name             = "ebs-csi"
+  name                  = "ebs-csi"
   attach_ebs_csi_policy = true
 
   oidc_providers = {
     ex = {
       provider_arn               = module.eks.oidc_provider_arn
-      namespace_service_accounts = ["consul:server", "kube-system:ebs-csi-controller-sa"]
+      namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
     }
   }
+  tags = var.tags
 }

aws/k8s-aws-alb-script-cleanup.sh

@@ -28,24 +28,4 @@ echo "cleanup k8s ingress and service. This may take a while"
 kubectl delete service secret-challenge
 kubectl delete ingress wrongsecrets
 
-echo "Cleanup helm chart"
-helm uninstall aws-load-balancer-controller \
-  -n kube-system
-
-echo "Cleanup k8s ALB"
-kubectl delete -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/crds?ref=master"
-
-echo "Cleanup iam serviceaccount and policy"
-eksctl delete iamserviceaccount \
-  --cluster $CLUSTERNAME \
-  --name aws-load-balancer-controller \
-  --namespace kube-system \
-  --region $AWS_REGION
-
-sleep 5 # Prevents race condition - command below may error out because it's still 'attached'
-
-aws iam delete-policy \
-  --policy-arn arn:aws:iam::${ACCOUNT_ID}:policy/AWSLoadBalancerControllerIAMPolicy
-
-echo "Wait for 10 seconds to let the AWS resources be cleaned up"
 sleep 10

aws/k8s-aws-alb-script.sh

@@ -5,89 +5,11 @@
 
 source ../scripts/check-available-commands.sh
 
-checkCommandsAvailable aws cat docker eksctl grep helm jq kubectl sed terraform vault
+checkCommandsAvailable kubectl
 
-if test -n "${AWS_REGION-}"; then
-  echo "AWS_REGION is set to <$AWS_REGION>"
-else
-  AWS_REGION=eu-west-1
-  echo "AWS_REGION is not set or empty, defaulting to ${AWS_REGION}"
-fi
-
-if test -n "${CLUSTERNAME-}"; then
-  echo "CLUSTERNAME is set to <$CLUSTERNAME>"
-else
-  CLUSTERNAME=wrongsecrets-exercise-cluster
-  echo "CLUSTERNAME is not set or empty, defaulting to ${CLUSTERNAME}"
-fi
-
-ACCOUNT_ID=$(aws sts get-caller-identity | jq '.Account' -r)
-echo "ACCOUNT_ID=${ACCOUNT_ID}"
-
-LBC_VERSION="v2.13.4"
-echo "LBC_VERSION=$LBC_VERSION"
-
-# echo "executing eksctl utils associate-iam-oidc-provider"
-# eksctl utils associate-iam-oidc-provider \
-#     --region ${AWS_REGION} \
-#     --cluster ${CLUSTERNAME} \
-#     --approve
-
-echo "creating iam policy"
-curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/"${LBC_VERSION}"/docs/install/iam_policy.json
-aws iam create-policy \
-  --policy-name AWSLoadBalancerControllerIAMPolicy \
-  --policy-document file://iam_policy.json
-
-echo "creating iam service account for cluster ${CLUSTERNAME}"
-eksctl create iamserviceaccount \
-  --cluster $CLUSTERNAME \
-  --namespace kube-system \
-  --name aws-load-balancer-controller \
-  --attach-policy-arn arn:aws:iam::${ACCOUNT_ID}:policy/AWSLoadBalancerControllerIAMPolicy \
-  --override-existing-serviceaccounts \
-  --region $AWS_REGION \
-  --approve
-
-echo "setting up kubectl"
-
-aws eks update-kubeconfig --region $AWS_REGION --name $CLUSTERNAME --kubeconfig ~/.kube/wrongsecrets
-
-export KUBECONFIG=~/.kube/wrongsecrets
-
-echo "applying aws-lbc with kubectl"
-
-kubectl apply -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller/crds?ref=master"
-
-kubectl get crd
-
-echo "do helm eks application"
-helm repo add eks https://aws.github.io/eks-charts
-helm repo update eks
-
-echo "upgrade alb controller with helm"
-helm upgrade -i aws-load-balancer-controller \
-  eks/aws-load-balancer-controller \
-  -n kube-system \
-  --set clusterName=${CLUSTERNAME} \
-  --set serviceAccount.create=false \
-  --set serviceAccount.name=aws-load-balancer-controller \
-  --set image.tag="${LBC_VERSION}" \
-  --set region=${AWS_REGION} \
-  --set image.repository=602401143452.dkr.ecr.${AWS_REGION}.amazonaws.com/amazon/aws-load-balancer-controller
-# You may need to modify the account ID above if you're operating in af-south-1, ap-east-1, ap-southeast-3, cn-north and cn-northwest, eu-south-1, me-south-1, or the govcloud.
-# See the full list of accounts per regions here: https://docs.aws.amazon.com/eks/latest/userguide/add-ons-images.html
-
-echo "wait with rollout for 10 s"
-sleep 10
-
-echo "rollout status deployment"
-kubectl -n kube-system rollout status deployment aws-load-balancer-controller
-
-echo "wait after rollout for 10 s"
-sleep 10
-
-EKS_CLUSTER_VERSION=$(aws eks describe-cluster --name $CLUSTERNAME --region $AWS_REGION --query cluster.version --output text)
+echo "set up ingress class"
+kubectl apply -f ./k8s/ingress-class-params.yaml
+kubectl apply -f ./k8s/ingress-class.yaml
 
 echo "apply -f k8s/secret-challenge-vault-service.yml in 10 s"
 sleep 10

aws/k8s-vault-aws-start.sh

@@ -17,7 +17,7 @@ fi
 if test -n "${CLUSTERNAME-}"; then
   echo "CLUSTERNAME is set to <$CLUSTERNAME>"
 else
-  CLUSTERNAME=wrongsecrets-exercise-cluster
+  CLUSTERNAME=wrongsecrets
   echo "CLUSTERNAME is not set or empty, defaulting to ${CLUSTERNAME}"
 fi
 
@@ -61,18 +61,21 @@ else
   kubectl apply -f ../k8s/challenge33.yml
 fi
 
-helm list -n | grep 'aws-ebs-csi-driver' &> /dev/null
-if [ $? == 0 ]; then
-  echo "AWS EBS CSI driver is already installed"
-else
-  echo "Installing AWS EBS CSI driver"
-  helm repo add aws-ebs-csi-driver https://kubernetes-sigs.github.io/aws-ebs-csi-driver
-  helm repo update
-  helm upgrade --install aws-ebs-csi-driver --version 2.32.0 \
-    --namespace kube-system \
-    aws-ebs-csi-driver/aws-ebs-csi-driver \
-    --values ./k8s/ebs-csi-driver-values.yaml
-fi
+#helm list -n | grep 'aws-ebs-csi-driver' &>/dev/null
+#if [ $? == 0 ]; then
+#  echo "AWS EBS CSI driver is already installed"
+#else
+#  echo "Installing AWS EBS CSI driver"
+#  helm repo add aws-ebs-csi-driver https://kubernetes-sigs.github.io/aws-ebs-csi-driver
+#  helm repo update
+#  helm upgrade --install aws-ebs-csi-driver --version 2.32.0 \
+#    --namespace kube-system \
+#    aws-ebs-csi-driver/aws-ebs-csi-driver \
+#    --values ./k8s/ebs-csi-driver-values.yaml
+#fi
+
+echo "Setting up gp3 storage class..."
+kubectl apply -f ./k8s/ebs-csi-gp3.yaml
 
 source ../scripts/install-vault.sh
 

aws/k8s/ebs-csi-driver-values.yaml

@@ -0,0 +1,21 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: gp3
+  annotations:
+    storageclass.kubernetes.io/is-default-class: "true"
+allowedTopologies:
+- matchLabelExpressions:
+  - key: eks.amazonaws.com/compute-type
+    values:
+    - auto
+provisioner: ebs.csi.eks.amazonaws.com
+volumeBindingMode: WaitForFirstConsumer
+reclaimPolicy: Delete
+allowVolumeExpansion: true
+  # The following parameters are specific to the EBS CSI driver.
+parameters:
+  type: gp3
+  encrypted: "true"
+  tagSpecification_1: Application=wrongsecrets
+

aws/k8s/ebs-csi-gp3.yaml

@@ -0,0 +1,6 @@
+apiVersion: eks.amazonaws.com/v1
+kind: IngressClassParams
+metadata:
+  name: alb
+spec:
+  scheme: internet-facing

aws/k8s/ingress-class-params.yaml

@@ -0,0 +1,12 @@
+apiVersion: networking.k8s.io/v1
+kind: IngressClass
+metadata:
+  name: alb
+  annotations:
+    ingressclass.kubernetes.io/is-default-class: "true"
+spec:
+  controller: eks.amazonaws.com/alb
+  parameters:
+    apiGroup: eks.amazonaws.com
+    kind: IngressClassParams
+    name: alb

aws/k8s/ingress-class.yaml

@@ -67,65 +67,29 @@ module "eks" {
   source  = "terraform-aws-modules/eks/aws"
   version = "21.3.1"
 
-  cluster_name    = var.cluster_name
-  cluster_version = var.cluster_version
+  name               = var.cluster_name
+  kubernetes_version = var.cluster_version
 
   vpc_id     = module.vpc.vpc_id
   subnet_ids = module.vpc.private_subnets
 
 
-  cluster_endpoint_private_access = true
-  cluster_endpoint_public_access  = true
+  endpoint_private_access = true
+  endpoint_public_access  = true
 
-  cluster_endpoint_public_access_cidrs = ["${data.http.ip.response_body}/32"]
+  endpoint_public_access_cidrs = ["${data.http.ip.response_body}/32"]
 
-  enable_irsa = true
+  #create_auto_mode_iam_resources = true
 
   enable_cluster_creator_admin_permissions = true
 
-  eks_managed_node_group_defaults = {
-    disk_size       = 50
-    disk_type       = "gp3"
-    disk_throughput = 150
-    disk_iops       = 3000
-    instance_types  = ["t3.large"]
-
-    iam_role_additional_policies = {
-      AmazonEKSWorkerNodePolicy : "arn:aws:iam::aws:policy/AmazonEKSWorkerNodePolicy",
-      AmazonEKS_CNI_Policy : "arn:aws:iam::aws:policy/AmazonEKS_CNI_Policy",
-      AmazonEC2ContainerRegistryReadOnly : "arn:aws:iam::aws:policy/AmazonEC2ContainerRegistryReadOnly",
-      AmazonSSMManagedInstanceCore : "arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore",
-      AmazonEKSVPCResourceController : "arn:aws:iam::aws:policy/AmazonEKSVPCResourceController",
-      AmazonEBSCSIDriverPolicy : "arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy"
-    }
+  upgrade_policy = {
+    support_type = "STANDARD"
   }
 
-  eks_managed_node_groups = {
-    bottlerocket_default = {
-      use_custom_launch_template = false
-      min_size                   = 1
-      max_size                   = 3
-      desired_size               = 1
-      capacity_type              = "SPOT"
-
-      ami_type = "BOTTLEROCKET_x86_64"
-      platform = "bottlerocket"
-    }
-  }
-
-  node_security_group_additional_rules = {
-    aws_lb_controller_webhook = {
-      description                   = "Cluster API to AWS LB Controller webhook"
-      protocol                      = "all"
-      from_port                     = 9443
-      to_port                       = 9443
-      type                          = "ingress"
-      source_cluster_security_group = true
-    }
-  }
-
-  tags = {
-    Environment = "test"
-    Application = "wrongsecrets"
+  compute_config = {
+    enabled    = true
+    node_pools = ["general-purpose", "system"]
   }
+  tags = var.tags
 }

aws/main.tf

@@ -49,6 +49,7 @@ POLICY
 resource "aws_secretsmanager_secret" "secret_2" {
   name                    = "wrongsecret-2"
   recovery_window_in_days = 0
+  tags                    = var.tags
 }
 
 resource "aws_secretsmanager_secret_policy" "policy_2" {