enh: #319 updated k8s version, updated dependent resources and small bugfixes (#324)

Author: khalezin

terraform/layer1-aws/README.md

@@ -8,6 +8,8 @@ resource "aws_cloudtrail" "main" {
   is_multi_region_trail         = true
 
   tags = local.tags
+
+  depends_on = [aws_s3_bucket_policy.cloudtrail]
 }
 
 #tfsec:ignore:aws-s3-enable-bucket-logging tfsec:ignore:aws-s3-enable-versioning tfsec:ignore:aws-cloudtrail-require-bucket-access-logging

terraform/layer1-aws/aws-cloudtrail.tf

@@ -3,8 +3,27 @@ locals {
     "k8s.io/cluster-autoscaler/enabled"       = "true"
     "k8s.io/cluster-autoscaler/${local.name}" = "owned"
   }
-  eks_addon_vpc_cni = merge(var.eks_addons.vpc-cni, { service_account_role_arn = module.vpc_cni_irsa.iam_role_arn })
-  eks_addons        = merge(var.eks_addons, { vpc-cni = local.eks_addon_vpc_cni })
+
+  eks_addons = merge({
+    vpc-cni = {
+      resolve_conflicts        = "OVERWRITE"
+      addon_version            = data.aws_eks_addon_version.vpc_cni.version
+      service_account_role_arn = module.vpc_cni_irsa.iam_role_arn
+    },
+    aws-ebs-csi-driver = {
+      resolve_conflicts        = "OVERWRITE"
+      addon_version            = data.aws_eks_addon_version.aws_ebs_csi_driver.version
+      service_account_role_arn = module.aws_ebs_csi_driver.iam_role_arn
+    },
+    coredns = {
+      resolve_conflicts = "OVERWRITE"
+      addon_version     = data.aws_eks_addon_version.coredns.version
+    },
+    kube-proxy = {
+      resolve_conflicts = "OVERWRITE"
+      addon_version     = data.aws_eks_addon_version.kube_proxy.version
+    }
+  })
 
   eks_map_roles = [
     {
@@ -247,6 +266,23 @@ module "vpc_cni_irsa" {
   tags = local.tags
 }
 
+module "aws_ebs_csi_driver" {
+  source  = "terraform-aws-modules/iam/aws//modules/iam-role-for-service-accounts-eks"
+  version = "4.14.0"
+
+  role_name             = "${local.name}-aws-ebs-csi-driver"
+  attach_ebs_csi_policy = true
+
+  oidc_providers = {
+    main = {
+      provider_arn               = module.eks.oidc_provider_arn
+      namespace_service_accounts = ["kube-system:ebs-csi-controller-sa"]
+    }
+  }
+
+  tags = local.tags
+}
+
 resource "aws_kms_key" "eks" {
   count       = var.eks_cluster_encryption_config_enable ? 1 : 0
   description = "EKS Secret Encryption Key"
@@ -255,3 +291,23 @@ resource "aws_kms_key" "eks" {
 resource "kubectl_manifest" "aws_auth_configmap" {
   yaml_body = local.aws_auth_configmap_yaml
 }
+
+data "aws_eks_addon_version" "aws_ebs_csi_driver" {
+  addon_name         = "aws-ebs-csi-driver"
+  kubernetes_version = var.eks_cluster_version
+}
+
+data "aws_eks_addon_version" "coredns" {
+  addon_name         = "coredns"
+  kubernetes_version = var.eks_cluster_version
+}
+
+data "aws_eks_addon_version" "kube_proxy" {
+  addon_name         = "kube-proxy"
+  kubernetes_version = var.eks_cluster_version
+}
+
+data "aws_eks_addon_version" "vpc_cni" {
+  addon_name         = "vpc-cni"
+  kubernetes_version = var.eks_cluster_version
+}

terraform/layer1-aws/aws-eks.tf

@@ -108,28 +108,10 @@ variable "single_nat_gateway" {
 
 # EKS
 variable "eks_cluster_version" {
-  default     = "1.22"
+  default     = "1.25"
   description = "Version of the EKS K8S cluster"
 }
 
-variable "eks_addons" {
-  default = {
-    coredns = {
-      resolve_conflicts = "OVERWRITE"
-      addon_version     = "v1.8.7-eksbuild.1"
-    }
-    kube-proxy = {
-      resolve_conflicts = "OVERWRITE"
-      addon_version     = "v1.22.6-eksbuild.1"
-    }
-    vpc-cni = {
-      resolve_conflicts = "OVERWRITE"
-      addon_version     = "v1.11.0-eksbuild.1"
-    }
-  }
-  description = "A list of installed EKS add-ons"
-}
-
 variable "eks_workers_additional_policies" {
   type        = list(any)
   default     = ["arn:aws:iam::aws:policy/AmazonSSMManagedInstanceCore"]

terraform/layer1-aws/variables.tf

@@ -8,6 +8,10 @@ locals {
     namespace     = local.helm_releases[index(local.helm_releases.*.id, "aws-node-termination-handler")].namespace
   }
   aws_node_termination_handler_values = <<VALUES
+rbac:
+  create: true
+  pspEnabled: false # Due to psp removed in k8s 1.25 and latest aws-node-termination handler chart doesn't maintain new PSP version
+
 enableSpotInterruptionDraining: true
 enableRebalanceMonitoring: true
 

terraform/layer2-k8s/README.md

@@ -46,7 +46,7 @@ affinity:
 resources:
   limits:
     cpu: 100m
-    memory: 512Mi
+    memory: 1024Mi
   requests:
     cpu: 100m
     memory: 320Mi
@@ -147,14 +147,14 @@ module "aws_iam_autoscaler" {
           "autoscaling:DescribeLaunchConfigurations",
           "autoscaling:DescribeAutoScalingInstances",
           "autoscaling:DescribeAutoScalingGroups",
+          "ec2:DescribeInstanceTypes",
         ],
         "Resource" : "*"
       },
       {
         "Sid" : "clusterAutoscalerOwn",
         "Effect" : "Allow",
         "Action" : [
-          "autoscaling:UpdateAutoScalingGroup",
           "autoscaling:TerminateInstanceInAutoScalingGroup",
           "autoscaling:SetDesiredCapacity",
         ],

terraform/layer2-k8s/eks-aws-node-termination-handler.tf

@@ -10,6 +10,9 @@ locals {
   loki_stack_values = <<VALUES
 loki:
   enabled: true
+  rbac:
+    create: true
+    pspEnabled: false # Due to psp removed in k8s 1.25 and latest loki-stack chart doesn't maintain new PSP version
   config:
     limits_config:
       enforce_metric_name: false

terraform/layer2-k8s/eks-cluster-autoscaler.tf

@@ -3,13 +3,13 @@ releases:
     enabled: true
     chart: aws-load-balancer-controller
     repository: https://aws.github.io/eks-charts
-    chart_version: 1.4.5
+    chart_version: 1.4.8
     namespace: aws-load-balancer-controller
   - id: aws-node-termination-handler
     enabled: true
     chart: aws-node-termination-handler
     repository: https://aws.github.io/eks-charts
-    chart_version: 0.19.3
+    chart_version: 0.21.0
     namespace: aws-node-termination-handler
   - id: cert-manager
     enabled: false
@@ -33,7 +33,7 @@ releases:
     enabled: true
     chart: cluster-autoscaler
     repository: https://kubernetes.github.io/autoscaler
-    chart_version: 9.13.1
+    chart_version: 9.26.0
     namespace: cluster-autoscaler
   - id: elk
     enabled: false
@@ -93,7 +93,7 @@ releases:
     enabled: true
     chart: loki-stack
     repository: https://grafana.github.io/helm-charts
-    chart_version: 2.8.3
+    chart_version: 2.9.9
     namespace: loki
   - id: reloader
     enabled: true

terraform/layer2-k8s/eks-loki-stack.tf

@@ -103,5 +103,5 @@ variable "nginx_ingress_ssl_terminator" {
 # Cluster autoscaler
 variable "cluster_autoscaler_version" {
   description = "Version of cluster autoscaler"
-  default     = "v1.22.0"
+  default     = "v1.25.0"
 }