add patch for basic modules for applying node selectors and tolerations

Author: Friz-zy

README.md

@@ -7,11 +7,13 @@ Setup basic EKS cluster with necessary controllers. Examples for further configu
 
 ## Depend on
 - terraform
-- helm
+- aws cli
 - kubectl
 - [terraform-aws-eks](https://github.com/terraform-aws-modules/terraform-aws-eks)
 - [aws-ia/eks-blueprints-addons/aws](https://github.com/aws-ia/terraform-aws-eks-blueprints-addons)
 
+This module contain local-exec block with `kubectl patch` for applying `tolerations` and `nodeSelector` deployments in `kube-system` namespace, that will work only in unix shell, so it will fail on Windows. This patch is necessary as some of eks addons currently doesn't support `tolerations` and `nodeSelector` in their configurations, but only necessary if you will use host nodes with taints to separate `management` processes from other. You can disable it by set `apply_kubectl_patch` variable to `false`.
+
 ## Example
 ```
 cd example
@@ -21,6 +23,8 @@ terraform apply
 terraform output all
 ```
 
+After `terraform destroy` check ec2 volumes for unused disks as aws-ebs-csi-driver doesn't delete it by default after deleting helm releases.
+
 ## Security
 
 `victoria-metrics-k8s-stack` deployed without internal password protection. Multiple charts such as `apisix`, `qryn` and `uptrace` contain explicit passwords in the values and do not use k8s secrets.

example/main.tf

@@ -8,6 +8,8 @@ locals {
   ingress_domain      = "cluster.local"
   cert_manager_issuer = ""
 
+  cluster_version     = "1.29"
+
   vpc_name = "eks_${local.cluster_name}"
   azs = data.aws_availability_zones.available.names
   cidr = "10.0.0.0/16"
@@ -129,6 +131,7 @@ module "eks" {
   source = "../"
 
   cluster_name                                    = local.cluster_name
+  cluster_version                                 = local.cluster_version
   admin_email                                     = local.admin_email
   ingress_domain                                  = local.ingress_domain
   cert_manager_issuer                             = local.cert_manager_issuer
@@ -145,22 +148,23 @@ module "eks" {
   fargate_profiles                                = local.fargate_profiles
   tags                                            = local.tags
 
-  enable_aws_efs_csi_driver       = true
-  enable_cert_manager             = true
-  enable_cluster_autoscaler       = true
-  enable_metrics_server           = true
-  enable_vpa                      = true
-  enable_ingress_apisix           = true
-  enable_victoriametrics_operator = true
-  enable_opentelemetry_operator   = true
-  enable_clickhouse_operator      = true
-  enable_grafana_operator         = true
-  enable_victoriametrics          = true
-  enable_grafana                  = true
-  enable_uptrace                  = true
-  enable_vector_agent             = true
-  enable_qryn                     = false
-  enable_openobserve              = false
-  enable_openobserve_collector    = false
-  enable_kubernetes_dashboard     = false
+  enable_aws_efs_csi_driver           = true
+  enable_aws_node_termination_handler = false
+  enable_cert_manager                 = true
+  enable_cluster_autoscaler           = true
+  enable_metrics_server               = true
+  enable_vpa                          = true
+  enable_ingress_apisix               = true
+  enable_victoriametrics_operator     = true
+  enable_opentelemetry_operator       = true
+  enable_clickhouse_operator          = true
+  enable_grafana_operator             = true
+  enable_victoriametrics              = true
+  enable_grafana                      = true
+  enable_uptrace                      = true
+  enable_vector_agent                 = true
+  enable_qryn                         = false
+  enable_openobserve                  = false
+  enable_openobserve_collector        = false
+  enable_kubernetes_dashboard         = false
 }

main.tf

@@ -227,12 +227,7 @@ locals {
 
   universal_cluster_addon_config = {
     most_recent = true
-    configuration_values = jsonencode({
-      nodeSelector = {
-        "kubernetes.io/os" = "linux"
-        "node.kubernetes.io/purpose" = "management"
-      }
-    })
+    configuration_values = jsonencode(yamldecode(file("${path.module}/universal_values.yaml")))
   }
 
   cluster_addons = merge(
@@ -256,21 +251,71 @@ locals {
     var.cluster_addons
   )
 
+  universal_values_string = templatefile("${path.module}/universal_values.yaml",  {})
   universal_addon_config = {
-    values = [templatefile("${path.module}/universal_values.yaml", {})]
+    values = [local.universal_values_string]
   }
 
-  aws_efs_csi_driver_config = merge(local.universal_addon_config, var.aws_efs_csi_driver_config)
+  # https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/charts/aws-efs-csi-driver/values.yaml
+  aws_efs_csi_driver_config = merge(
+    local.universal_addon_config,
+    {
+      values = [
+        <<-EOT
+          controller:
+            ${replace(local.universal_values_string, "\n", "\n  ")}
+        EOT
+      ]
+    },
+    var.aws_efs_csi_driver_config
+  )
 
-  # aws_node_termination_handler_config = merge(local.universal_addon_config, var.aws_node_termination_handler_config)
+  # https://github.com/aws/aws-node-termination-handler/blob/main/config/helm/aws-node-termination-handler/values.yaml
+  aws_node_termination_handler_config = merge(local.universal_addon_config, var.aws_node_termination_handler_config)
 
-  cert_manager_config = merge(local.universal_addon_config, var.cert_manager_config)
+  # https://github.com/cert-manager/cert-manager/blob/master/deploy/charts/cert-manager/values.yaml
+  cert_manager_config = merge(
+    local.universal_addon_config,
+    {
+      values = [
+        <<-EOT
+          webhook:
+            ${replace(local.universal_values_string, "\n", "\n  ")}
+          cainjector:
+            ${replace(local.universal_values_string, "\n", "\n  ")}
+          startupapicheck:
+            ${replace(local.universal_values_string, "\n", "\n  ")}
+        EOT
+      ]
+    },
+    var.cert_manager_config
+  )
 
+  # https://github.com/kubernetes/autoscaler/blob/master/charts/cluster-autoscaler/values.yaml
   cluster_autoscaler_config = merge(local.universal_addon_config, var.cluster_autoscaler_config)
 
+  # https://github.com/kubernetes-sigs/metrics-server/blob/master/charts/metrics-server/values.yaml
   metrics_server_config = merge(local.universal_addon_config, var.metrics_server_config)
 
-  vpa_config = merge(local.universal_addon_config, var.vpa_config)
+  # https://github.com/FairwindsOps/charts/blob/master/stable/vpa/values.yaml
+  vpa_config = merge(
+    local.universal_addon_config,
+    {
+      values = [
+        <<-EOT
+          recommender:
+            ${replace(local.universal_values_string, "\n", "\n  ")}
+          updater:
+            ${replace(local.universal_values_string, "\n", "\n  ")}
+          admissionController:
+            ${replace(local.universal_values_string, "\n", "\n  ")}
+          mutatingWebhookConfiguration:
+            ${replace(local.universal_values_string, "\n", "\n  ")}
+        EOT
+      ]
+    },
+    var.vpa_config
+  )
 
   # don't like using root password for monitoring agents but for speedup
   openobserve_authorization = try(base64encode("${var.admin_email}:${module.openobserve.zo_root_user_password}"), "")
@@ -350,33 +395,56 @@ module "addons" {
   eks_addons = local.cluster_addons
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/aws-efs-csi-driver.md
+  # https://github.com/kubernetes-sigs/aws-efs-csi-driver/blob/master/charts/aws-efs-csi-driver/values.yaml
   enable_aws_efs_csi_driver = var.enable_aws_efs_csi_driver
   aws_efs_csi_driver = local.aws_efs_csi_driver_config
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/aws-node-termination-handler.md
-  #enable_aws_node_termination_handler = var.enable_aws_node_termination_handler
-  #aws_node_termination_handler = local.aws_node_termination_handler_config
+  # https://github.com/aws/aws-node-termination-handler/blob/main/config/helm/aws-node-termination-handler/values.yaml
+  enable_aws_node_termination_handler = var.enable_aws_node_termination_handler
+  aws_node_termination_handler = local.aws_node_termination_handler_config
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/cert-manager.md
   # https://github.com/cert-manager/cert-manager/blob/master/deploy/charts/cert-manager/values.yaml
   enable_cert_manager = var.enable_cert_manager
   cert_manager = local.cert_manager_config
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/cluster-autoscaler.md
+  # https://github.com/kubernetes/autoscaler/blob/master/charts/cluster-autoscaler/values.yaml
   enable_cluster_autoscaler = var.enable_cluster_autoscaler
   cluster_autoscaler = local.cluster_autoscaler_config
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/metrics-server.md
+  # https://github.com/kubernetes-sigs/metrics-server/blob/master/charts/metrics-server/values.yaml
   enable_metrics_server = var.enable_metrics_server
   metrics_server = local.metrics_server_config
 
   # https://github.com/aws-ia/terraform-aws-eks-blueprints-addons/blob/0e9d6c9b7115ecf0404c377c9c2529bffa56d10d/docs/addons/vertical-pod-autoscaler.md
+  # https://github.com/FairwindsOps/charts/blob/master/stable/vpa/values.yaml
   enable_vpa = var.enable_vpa
   vpa = local.vpa_config
 
   tags = var.tags
 }
 
+# patch addons and modules as some eks addons don't have tolerations
+resource "null_resource" "apply_kubectl_patch" {
+
+  count = var.apply_kubectl_patch? 1 : 0
+
+  depends_on = [
+    #module.eks,
+    module.addons
+  ]
+
+  provisioner "local-exec" {
+    command = <<-EOT
+      export KUBECONFIG="${module.eks.kubeconfig}"
+      kubectl get deployments -o name -n kube-system | xargs -I {} kubectl patch {} -n kube-system -p '{"spec": {"template":{"spec":${jsonencode(yamldecode(file("${path.module}/universal_values.yaml")))}}}}'
+    EOT
+  }
+}
+
 # https://cert-manager.io/docs/configuration/acme/
 module "cert_manager_acme_manifests" {
   source = "./modules/kubernetes-manifests"
@@ -445,7 +513,7 @@ module "victoriametrics_operator" {
   tags          = var.tags
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     var.victoriametrics_operator_values
   )
 }
@@ -466,7 +534,7 @@ module "opentelemetry_operator" {
   tags          = var.tags
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     var.opentelemetry_operator_values
   )
 }
@@ -487,7 +555,7 @@ module "grafana_operator" {
   tags          = var.tags
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     var.grafana_operator_values
   )
 }
@@ -508,7 +576,7 @@ module "clickhouse_operator" {
   tags          = var.tags
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     var.clickhouse_operator_values
   )
 }
@@ -532,7 +600,7 @@ module "ingress_apisix" {
   tags          = var.tags
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     [
     <<-EOT
       %{ if var.enable_victoriametrics_operator == true }
@@ -572,7 +640,7 @@ module "victoriametrics" {
   grafana_operator_namespace    = var.grafana_operator_namespace
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     [
     <<-EOT
       %{ if var.enable_victoriametrics_operator == true }
@@ -647,7 +715,7 @@ module "victoriametrics" {
   auth_chart_version = var.victoriametrics_auth_chart_version
   auth_set           = var.victoriametrics_auth_set
   auth_values        = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     [
     <<-EOT
       %{ if var.victoriametrics_auth_ingress_enabled == true }
@@ -719,7 +787,7 @@ module "grafana" {
   grafana_operator_namespace    = var.grafana_operator_namespace
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     [
     <<-EOT
       %{ if var.enable_victoriametrics_operator == true }
@@ -778,7 +846,7 @@ module "uptrace" {
   grafana_operator_namespace    = var.grafana_operator_namespace
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     [
     <<-EOT
       %{ if var.uptrace_ingress_enabled == true }
@@ -819,15 +887,15 @@ module "uptrace" {
   clickhouse_set           = var.uptrace_clickhouse_set
 
   clickhouse_values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     var.uptrace_clickhouse_values
   )
 
   postgresql_chart_version = var.uptrace_postgresql_chart_version
   postgresql_set           = var.uptrace_postgresql_set
 
   postgresql_values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     var.uptrace_postgresql_values
   )
 }
@@ -854,7 +922,7 @@ module "qryn" {
   grafana_operator_namespace    = var.grafana_operator_namespace
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     [
     <<-EOT
       %{ if var.enable_victoriametrics_operator == true }
@@ -895,7 +963,7 @@ module "qryn" {
   clickhouse_set           = var.qryn_clickhouse_set
 
   clickhouse_values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     var.qryn_clickhouse_values
   )
 }
@@ -920,7 +988,7 @@ module "openobserve" {
   oidc_provider_arn     = module.eks.oidc_provider_arn
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     [
     <<-EOT
       %{ if var.openobserve_ingress_enabled == true }
@@ -974,7 +1042,7 @@ module "openobserve_collector" {
   zo_authorization = "Basic ${local.openobserve_authorization}"
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     var.openobserve_collector_values
   )
 }
@@ -992,7 +1060,7 @@ module "vector_agent" {
   tags          = var.tags
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     [
     <<-EOT
       role: "Agent"
@@ -1074,7 +1142,7 @@ module "kubernetes_dashboard" {
   tags          = var.tags
 
   values = concat(
-    [templatefile("${path.module}/universal_values.yaml", {})],
+    [local.universal_values_string],
     [
     <<-EOT
     %{ if var.kubernetes_dashboard_ingress_enabled == true }