fix: several fixes related to service accounts

Fix EBS CSI driver SA
Fix load balancer controller SA
Clean up scripts and tf

Author: bendehaan

aws/build-an-deploy-aws.sh

@@ -53,6 +53,10 @@ echo "CLUSTER_AUTOSCALER_ROLE_ARN=${CLUSTER_AUTOSCALER_ROLE_ARN}"
 
 version="$(uuidgen)"
 
+aws eks update-kubeconfig --region $AWS_REGION --name $CLUSTERNAME --kubeconfig ~/.kube/wrongsecrets
+
+export KUBECONFIG=~/.kube/wrongsecrets
+
 echo "If the below output is different than expected: please hard stop this script (running aws sts get-caller-identity first)"
 
 aws sts get-caller-identity
@@ -61,24 +65,6 @@ echo "Giving you 4 seconds before we add autoscaling"
 
 sleep 4
 
-# echo "Installing policies and service accounts"
-
-# aws iam create-policy \
-#     --policy-name AmazonEKSClusterAutoscalerPolicy \
-#     --policy-document file://cluster-autoscaler-policy.json
-
-# echo "Installing iamserviceaccount"
-
-# eksctl create iamserviceaccount \
-#   --cluster=$CLUSTERNAME \
-#   --region=$AWS_REGION \
-#   --namespace=kube-system \
-#   --name=cluster-autoscaler \
-#   --role-name=AmazonEKSClusterAutoscalerRole \
-#   --attach-policy-arn=arn:aws:iam::${ACCOUNT_ID}:policy/AmazonEKSClusterAutoscalerPolicy \
-#   --override-existing-serviceaccounts \
-#   --approve
-
 echo "Deploying the k8s autoscaler for eks through kubectl"
 
 curl -o cluster-autoscaler-autodiscover.yaml https://raw.githubusercontent.com/kubernetes/autoscaler/master/cluster-autoscaler/cloudprovider/aws/examples/cluster-autoscaler-autodiscover.yaml
@@ -156,33 +142,16 @@ helm upgrade --install mj ../helm/wrongsecrets-ctf-party \
   --set="balancer.env.REACT_APP_CREATE_TEAM_HMAC_KEY=${CREATE_TEAM_HMAC}" \
   --set="balancer.cookie.cookieParserSecret=${COOKIE_PARSER_SECRET}"
 
-# echo "Installing EBS CSI driver"
-# eksctl create iamserviceaccount \
-#   --name ebs-csi-controller-sa \
-#   --namespace kube-system \
-#   --cluster $CLUSTERNAME \
-#   --attach-policy-arn arn:aws:iam::aws:policy/service-role/AmazonEBSCSIDriverPolicy \
-#   --approve \
-#   --role-only \
-#   --role-name AmazonEKS_EBS_CSI_DriverRole
-#   --region $AWS_REGION
-
-# echo "managing EBS CSI Driver as a separate eks addon"
-# eksctl create addon --name aws-ebs-csi-driver \
-#   --cluster $CLUSTERNAME \
-#   --service-account-role-arn arn:aws:iam::${ACCOUNT_ID}:role/AmazonEKS_EBS_CSI_DriverRole \
-#   --force \
-#   --region $AWS_REGION
-
 # Install CTFd
-
 echo "Installing CTFd"
 
 export HELM_EXPERIMENTAL_OCI=1
 kubectl create namespace ctfd
+
+# Double base64 encoding to prevent weird character errors in ctfd
 helm upgrade --install ctfd -n ctfd oci://ghcr.io/bman46/ctfd/ctfd \
-  --set="redis.auth.password=$(openssl rand -base64 24)" \
-  --set="mariadb.auth.rootPassword=$(openssl rand -base64 24)" \
-  --set="mariadb.auth.password=$(openssl rand -base64 24)" \
-  --set="mariadb.auth.replicationPassword=$(openssl rand -base64 24)" \
+  --set="redis.auth.password=$(openssl rand -base64 24 | base64)" \
+  --set="mariadb.auth.rootPassword=$(openssl rand -base64 24 | base64)" \
+  --set="mariadb.auth.password=$(openssl rand -base64 24 | base64)" \
+  --set="mariadb.auth.replicationPassword=$(openssl rand -base64 24 | base64)" \
   --set="env.open.SECRET_KEY=test" # this key isn't actually necessary in a setup with CTFd

aws/cleanup-aws-autoscaling-and-helm.sh

@@ -36,26 +36,3 @@ helm uninstall csi-secrets-store \
 echo "Cleanup helm chart projectcalico"
 helm uninstall calico \
   -n default
-
-echo "cleanup serviceaccont"
-echo "Cleanup iam serviceaccount and policy"
-eksctl delete iamserviceaccount \
-  --cluster $CLUSTERNAME \
-  --name cluster-autoscaler \
-  --namespace kube-system \
-  --region $AWS_REGION
-
-
-sleep 5 # Prevents race condition - command below may error out because it's still 'attached'
-
-aws iam delete-policy \
-  --policy-arn arn:aws:iam::${ACCOUNT_ID}:policy/AmazonEKSClusterAutoscalerPolicy
-
-
-echo "Cleanup CSI driver SA"
-
-eksctl delete iamserviceaccount \
-  --cluster $CLUSTERNAME \
-  --name ebs-csi-controller-sa \
-  --namespace kube-system \
-  --region $AWS_REGION

aws/k8s-aws-alb-script-cleanup.sh

@@ -34,15 +34,3 @@ helm uninstall aws-load-balancer-controller \
 
 echo "Cleanup k8s ALB"
 kubectl delete -k "github.com/aws/eks-charts/stable/aws-load-balancer-controller//crds?ref=master"
-
-echo "Cleanup iam serviceaccount and policy"
-eksctl delete iamserviceaccount \
-  --cluster $CLUSTERNAME \
-  --name aws-load-balancer-controller \
-  --namespace kube-system \
-  --region $AWS_REGION
-
-sleep 5 # Prevents race condition - command below may error out because it's still 'attached'
-
-aws iam delete-policy \
-  --policy-arn arn:aws:iam::${ACCOUNT_ID}:policy/AWSLoadBalancerControllerIAMPolicy

aws/k8s-aws-alb-script.sh

@@ -26,28 +26,6 @@ echo "ACCOUNT_ID=${ACCOUNT_ID}"
 LBC_VERSION="v2.4.1"
 echo "LBC_VERSION=$LBC_VERSION"
 
-# echo "executing eksctl utils associate-iam-oidc-provider"
-# eksctl utils associate-iam-oidc-provider \
-#     --region ${AWS_REGION} \
-#     --cluster ${CLUSTERNAME} \
-#     --approve
-
-echo "creating iam policy"
-curl -o iam_policy.json https://raw.githubusercontent.com/kubernetes-sigs/aws-load-balancer-controller/"${LBC_VERSION}"/docs/install/iam_policy.json
-aws iam create-policy \
-  --policy-name AWSLoadBalancerControllerIAMPolicy \
-  --policy-document file://iam_policy.json
-
-echo "creating iam service account for cluster ${CLUSTERNAME}"
-eksctl create iamserviceaccount \
-  --cluster $CLUSTERNAME \
-  --namespace kube-system \
-  --name aws-load-balancer-controller \
-  --attach-policy-arn arn:aws:iam::${ACCOUNT_ID}:policy/AWSLoadBalancerControllerIAMPolicy \
-  --override-existing-serviceaccounts \
-  --region $AWS_REGION \
-  --approve
-
 echo "setting up kubectl"
 
 aws eks update-kubeconfig --region $AWS_REGION --name $CLUSTERNAME --kubeconfig ~/.kube/wrongsecrets

aws/main.tf

@@ -72,7 +72,8 @@ module "eks" {
 
   cluster_addons = {
     aws-ebs-csi-driver = {
-      most_recent = true
+      most_recent              = true
+      service_account_role_arn = module.ebs_csi_irsa_role.iam_role_arn
     }
   }
 

aws/variables.tf

@@ -7,7 +7,7 @@ variable "region" {
 variable "cluster_version" {
   description = "The EKS cluster version to use"
   type        = string
-  default     = "1.23"
+  default     = "1.25"
 }
 
 variable "cluster_name" {