up node termination handler and other eks addons

Author: Friz-zy

README.md

@@ -23,6 +23,18 @@ terraform apply
 terraform output all
 ```
 
+to destroy everything run (you may need to run it twice one by one)
+```
+terraform destroy -auto-approve
+```
+
+force destroy in case of problems
+```
+helm ls -a --all-namespaces | awk 'NR > 1 { print  "-n "$2, $1}' | xargs -L1 helm delete
+kubectl delete all --all --all-namespaces
+terraform destroy -auto-approve
+```
+
 After `terraform destroy` check ec2 volumes for unused disks as aws-ebs-csi-driver doesn't delete it by default after deleting helm releases.
 
 ## Security

example/main.tf

@@ -6,6 +6,7 @@ locals {
   cluster_name        = "test"
   admin_email         = "[email protected]"
   ingress_domain      = "cluster.local"
+  ingress_class_name  = "apisix" # or "nginx"
   cert_manager_issuer = ""
 
   cluster_version     = "1.29"
@@ -41,6 +42,8 @@ locals {
         "node.kubernetes.io/purpose" = "management"
       }
 
+# multiple pods don't have tolerations yet
+# including snapshot-controller plugin that can't be changed at all
 #       taints = {
 #         purpose = {
 #           key    = "node.kubernetes.io/purpose"
@@ -149,7 +152,7 @@ module "eks" {
   tags                                            = local.tags
 
   enable_aws_efs_csi_driver           = true
-  enable_aws_node_termination_handler = false
+  enable_aws_node_termination_handler = true
   enable_cert_manager                 = true
   enable_cluster_autoscaler           = true
   enable_metrics_server               = true

main.tf

@@ -230,7 +230,7 @@ locals {
     configuration_values = jsonencode(yamldecode(file("${path.module}/universal_values.yaml")))
   }
 
-  cluster_addons = merge(
+  eks_addons = merge(
     {
       coredns = local.universal_cluster_addon_config
       kube-proxy = {
@@ -248,7 +248,7 @@ locals {
         most_recent = true
       }
     },
-    var.cluster_addons
+    var.eks_addons
   )
 
   universal_values_string = templatefile("${path.module}/universal_values.yaml",  {})
@@ -260,6 +260,7 @@ locals {
   aws_efs_csi_driver_config = merge(
     local.universal_addon_config,
     {
+      reset_values = true
       values = [
         <<-EOT
           controller:
@@ -271,12 +272,22 @@ locals {
   )
 
   # https://github.com/aws/aws-node-termination-handler/blob/main/config/helm/aws-node-termination-handler/values.yaml
-  aws_node_termination_handler_config = merge(local.universal_addon_config, var.aws_node_termination_handler_config)
+  aws_node_termination_handler_config = merge(
+    local.universal_addon_config,
+    {reset_values = true},
+    var.aws_node_termination_handler_config
+  )
+  aws_node_termination_handler_asg_arns = concat(
+    [for asg in module.eks.self_managed_node_groups : asg.autoscaling_group_arn],
+    [for asg in module.eks.eks_managed_node_groups : asg.autoscaling_group_arn],
+    var.aws_node_termination_handler_asg_arns
+  )
 
   # https://github.com/cert-manager/cert-manager/blob/master/deploy/charts/cert-manager/values.yaml
   cert_manager_config = merge(
     local.universal_addon_config,
     {
+      reset_values = true
       values = [
         <<-EOT
           webhook:
@@ -292,15 +303,24 @@ locals {
   )
 
   # https://github.com/kubernetes/autoscaler/blob/master/charts/cluster-autoscaler/values.yaml
-  cluster_autoscaler_config = merge(local.universal_addon_config, var.cluster_autoscaler_config)
+  cluster_autoscaler_config = merge(
+    local.universal_addon_config,
+    {reset_values = true},
+    var.cluster_autoscaler_config
+  )
 
   # https://github.com/kubernetes-sigs/metrics-server/blob/master/charts/metrics-server/values.yaml
-  metrics_server_config = merge(local.universal_addon_config, var.metrics_server_config)
+  metrics_server_config = merge(
+    local.universal_addon_config,
+    {reset_values = true},
+    var.metrics_server_config
+  )
 
   # https://github.com/FairwindsOps/charts/blob/master/stable/vpa/values.yaml
   vpa_config = merge(
     local.universal_addon_config,
     {
+      reset_values = true
       values = [
         <<-EOT
           recommender:
@@ -392,37 +412,41 @@ module "addons" {
   cluster_version   = module.eks.cluster_version
   oidc_provider_arn = module.eks.oidc_provider_arn
 
-  eks_addons = local.cluster_addons
+  eks_addons          = local.eks_addons
+  eks_addons_timeouts = var.eks_addons_timeouts
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/aws-efs-csi-driver.md
   # https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/charts/aws-efs-csi-driver/values.yaml
   enable_aws_efs_csi_driver = var.enable_aws_efs_csi_driver
-  aws_efs_csi_driver = local.aws_efs_csi_driver_config
+  aws_efs_csi_driver        = local.aws_efs_csi_driver_config
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/aws-node-termination-handler.md
   # https://github.com/aws/aws-node-termination-handler/blob/main/config/helm/aws-node-termination-handler/values.yaml
-  enable_aws_node_termination_handler = var.enable_aws_node_termination_handler
-  aws_node_termination_handler = local.aws_node_termination_handler_config
+  enable_aws_node_termination_handler   = var.enable_aws_node_termination_handler
+  aws_node_termination_handler          = local.aws_node_termination_handler_config
+  aws_node_termination_handler_sqs      = var.aws_node_termination_handler_sqs
+  aws_node_termination_handler_asg_arns = local.aws_node_termination_handler_asg_arns
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/cert-manager.md
   # https://github.com/cert-manager/cert-manager/blob/master/deploy/charts/cert-manager/values.yaml
-  enable_cert_manager = var.enable_cert_manager
-  cert_manager = local.cert_manager_config
+  enable_cert_manager                   = var.enable_cert_manager
+  cert_manager                          = local.cert_manager_config
+  cert_manager_route53_hosted_zone_arns = var.cert_manager_route53_hosted_zone_arns
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/cluster-autoscaler.md
   # https://github.com/kubernetes/autoscaler/blob/master/charts/cluster-autoscaler/values.yaml
   enable_cluster_autoscaler = var.enable_cluster_autoscaler
-  cluster_autoscaler = local.cluster_autoscaler_config
+  cluster_autoscaler        = local.cluster_autoscaler_config
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/metrics-server.md
   # https://github.com/kubernetes-sigs/metrics-server/blob/master/charts/metrics-server/values.yaml
   enable_metrics_server = var.enable_metrics_server
-  metrics_server = local.metrics_server_config
+  metrics_server        = local.metrics_server_config
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/vertical-pod-autoscaler.md
   # https://github.com/FairwindsOps/charts/blob/master/stable/vpa/values.yaml
   enable_vpa = var.enable_vpa
-  vpa = local.vpa_config
+  vpa        = local.vpa_config
 
   tags = var.tags
 }

modules/victoriametrics/main.tf

@@ -228,7 +228,7 @@ module "auth" {
   version = "~> 1.1"
 
   depends_on = [
-    #module.victoriametrics,
+    module.victoriametrics,
     #module.kubernetes_manifests
   ]