enh: use aws-load-balancer-controller in front of ingress-nginx (#293)

mglotov · web-flow · commit 65690441eb57 · 2022-05-26T11:59:39.000+06:00
diff --git a/README.md b/README.md
@@ -396,14 +396,17 @@ terragrunt destroy
 
 ## What to do after deployment
 
-After applying this configuration, you will get the infrastructure described and outlined at the beginning of the document. In AWS and within the EKS cluster, the basic resources and services necessary for the operation of the EKS k8s cluster will be created.
+* After applying this configuration, you will get the infrastructure described and outlined at the beginning of the document. In AWS and within the EKS cluster, the basic resources and services necessary for the operation of the EKS k8s cluster will be created.
 
-You can get access to the cluster using this command:
+* You can get access to the cluster using this command:
 
   ```bash
   aws eks update-kubeconfig --name maddevs-demo-use1 --region us-east-1
   ```
 
+* If you used default configuration and want to serve traffic for a main domain (example.com) by an application deployed into a k8s cluster, youn need to manually create DNS record in Route53 with type A + Alias
+* DNS record `*.example.com` created automatically and points to Load Balancer in front of k8s cluster. 
+
 ## Update terraform version
 
 Change terraform version in this files
diff --git a/docs/FAQ.md b/docs/FAQ.md
@@ -304,3 +304,10 @@ kubectl delete installations.operator.tigera.io default
 kubectl delete ns calico-apiserver calico-system
 ```
 5. Restart all nodes
+
+## What if you don't want to use an aws-load-balancer controller in front of an ingress-nginx and want to use a cert-manager and terminate SSL on ingres-nginx side
+
+1. Set `nginx ` for a `nginx_ingress_ssl_terminator` variable in the layer2-k8s folder
+2. Set `enabled: false` for `id: aws-load-balancer-controller` in the **layer2-k8s/helm-releases.yaml** file
+3. Set `enabled: true` for `id: external-dns`, `id: cert-manager`, `id: cert-mananger-certificate`, `id:cert-manager-cluster-issuer` in the **layer2-k8s/helm-releases.yaml** file
+4. Run `terraform apply` in the layer2-k8s folder
diff --git a/terraform/layer2-k8s/README.md b/terraform/layer2-k8s/README.md
@@ -21,6 +21,7 @@
 | <a name="provider_kubernetes"></a> [kubernetes](#provider\_kubernetes) | 2.10.0 |
 | <a name="provider_random"></a> [random](#provider\_random) | 3.1.3 |
 | <a name="provider_terraform"></a> [terraform](#provider\_terraform) | n/a |
+| <a name="provider_tls"></a> [tls](#provider\_tls) | 3.3.0 |
 
 ## Modules
 
@@ -58,6 +59,7 @@
 
 | Name | Type |
 |------|------|
+| [aws_route53_record.default_ingress](https://registry.terraform.io/providers/aws/4.10.0/docs/resources/route53_record) | resource |
 | [aws_s3_bucket.elastic_stack](https://registry.terraform.io/providers/aws/4.10.0/docs/resources/s3_bucket) | resource |
 | [aws_s3_bucket.gitlab_runner_cache](https://registry.terraform.io/providers/aws/4.10.0/docs/resources/s3_bucket) | resource |
 | [aws_s3_bucket_public_access_block.elastic_stack_public_access_block](https://registry.terraform.io/providers/aws/4.10.0/docs/resources/s3_bucket_public_access_block) | resource |
@@ -86,6 +88,7 @@
 | [kubectl_manifest.istio_prometheus_service_monitor_cp](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource |
 | [kubectl_manifest.istio_prometheus_service_monitor_dp](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource |
 | [kubectl_manifest.kube_prometheus_stack_operator_crds](https://registry.terraform.io/providers/gavinbunney/kubectl/1.14.0/docs/resources/manifest) | resource |
+| [kubernetes_ingress_v1.default](https://registry.terraform.io/providers/kubernetes/2.10.0/docs/resources/ingress_v1) | resource |
 | [kubernetes_secret.elasticsearch_certificates](https://registry.terraform.io/providers/kubernetes/2.10.0/docs/resources/secret) | resource |
 | [kubernetes_secret.elasticsearch_credentials](https://registry.terraform.io/providers/kubernetes/2.10.0/docs/resources/secret) | resource |
 | [kubernetes_secret.elasticsearch_s3_user_creds](https://registry.terraform.io/providers/kubernetes/2.10.0/docs/resources/secret) | resource |
@@ -96,6 +99,11 @@
 | [random_string.kibana_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
 | [random_string.kube_prometheus_stack_grafana_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
 | [random_string.victoria_metrics_k8s_stack_grafana_password](https://registry.terraform.io/providers/hashicorp/random/latest/docs/resources/string) | resource |
+| [tls_cert_request.aws_loadbalancer_controller_webhook](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/cert_request) | resource |
+| [tls_locally_signed_cert.aws_loadbalancer_controller_webhook](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/locally_signed_cert) | resource |
+| [tls_private_key.aws_loadbalancer_controller_webhook](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource |
+| [tls_private_key.aws_loadbalancer_controller_webhook_ca](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/private_key) | resource |
+| [tls_self_signed_cert.aws_loadbalancer_controller_webhook_ca](https://registry.terraform.io/providers/hashicorp/tls/latest/docs/resources/self_signed_cert) | resource |
 | [aws_caller_identity.current](https://registry.terraform.io/providers/aws/4.10.0/docs/data-sources/caller_identity) | data source |
 | [aws_eks_cluster.main](https://registry.terraform.io/providers/aws/4.10.0/docs/data-sources/eks_cluster) | data source |
 | [aws_eks_cluster_auth.main](https://registry.terraform.io/providers/aws/4.10.0/docs/data-sources/eks_cluster_auth) | data source |
diff --git a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf
@@ -7,7 +7,10 @@ locals {
     chart_version = local.helm_releases[index(local.helm_releases.*.id, "aws-load-balancer-controller")].chart_version
     namespace     = local.helm_releases[index(local.helm_releases.*.id, "aws-load-balancer-controller")].namespace
   }
-  aws_load_balancer_controller_values = <<VALUES
+  ssl_certificate_arn                               = data.terraform_remote_state.layer1-aws.outputs.ssl_certificate_arn
+  aws_load_balancer_controller_webhook_service_name = "${local.aws_load_balancer_controller.name}-webhook-service"
+  aws_load_balancer_controller_values               = <<VALUES
+nameOverride: ${local.aws_load_balancer_controller.name}
 clusterName: ${local.eks_cluster_id}
 region: ${local.region}
 vpcId: ${local.vpc_id}
@@ -115,12 +118,24 @@ module "aws_iam_aws_loadbalancer_controller" {
       {
         "Effect" : "Allow",
         "Action" : [
-          "iam:CreateServiceLinkedRole",
+          "iam:CreateServiceLinkedRole"
+        ],
+        "Resource" : "*",
+        "Condition" : {
+          "StringEquals" : {
+            "iam:AWSServiceName" : "elasticloadbalancing.amazonaws.com"
+          }
+        }
+      },
+      {
+        "Effect" : "Allow",
+        "Action" : [
           "ec2:DescribeAccountAttributes",
           "ec2:DescribeAddresses",
           "ec2:DescribeAvailabilityZones",
           "ec2:DescribeInternetGateways",
           "ec2:DescribeVpcs",
+          "ec2:DescribeVpcPeeringConnections",
           "ec2:DescribeSubnets",
           "ec2:DescribeSecurityGroups",
           "ec2:DescribeInstances",
@@ -318,6 +333,62 @@ module "aws_iam_aws_loadbalancer_controller" {
   })
 }
 
+resource "tls_private_key" "aws_loadbalancer_controller_webhook_ca" {
+  count = local.aws_load_balancer_controller.enabled ? 1 : 0
+
+  algorithm = "RSA"
+}
+
+resource "tls_self_signed_cert" "aws_loadbalancer_controller_webhook_ca" {
+  count = local.aws_load_balancer_controller.enabled ? 1 : 0
+
+  private_key_pem       = tls_private_key.aws_loadbalancer_controller_webhook_ca[count.index].private_key_pem
+  validity_period_hours = 87600 # 10 years
+  early_renewal_hours   = 8760  # 1 year
+  is_ca_certificate     = true
+  allowed_uses = [
+    "cert_signing",
+    "key_encipherment",
+    "digital_signature"
+  ]
+  subject {
+    common_name  = local.aws_load_balancer_controller_webhook_service_name
+    organization = local.name
+  }
+}
+
+resource "tls_private_key" "aws_loadbalancer_controller_webhook" {
+  count = local.aws_load_balancer_controller.enabled ? 1 : 0
+
+  algorithm = "RSA"
+}
+
+resource "tls_cert_request" "aws_loadbalancer_controller_webhook" {
+  count = local.aws_load_balancer_controller.enabled ? 1 : 0
+
+  private_key_pem = tls_private_key.aws_loadbalancer_controller_webhook[count.index].private_key_pem
+  dns_names       = ["${local.aws_load_balancer_controller_webhook_service_name}.${module.aws_load_balancer_controller_namespace[count.index].name}", "${local.aws_load_balancer_controller_webhook_service_name}.${module.aws_load_balancer_controller_namespace[count.index].name}.svc", "${local.aws_load_balancer_controller_webhook_service_name}.${module.aws_load_balancer_controller_namespace[count.index].name}.svc.cluster.local"]
+  subject {
+    common_name  = local.aws_load_balancer_controller_webhook_service_name
+    organization = local.name
+  }
+}
+
+resource "tls_locally_signed_cert" "aws_loadbalancer_controller_webhook" {
+  count = local.aws_load_balancer_controller.enabled ? 1 : 0
+
+  cert_request_pem   = tls_cert_request.aws_loadbalancer_controller_webhook[count.index].cert_request_pem
+  ca_private_key_pem = tls_private_key.aws_loadbalancer_controller_webhook_ca[count.index].private_key_pem
+  ca_cert_pem        = tls_self_signed_cert.aws_loadbalancer_controller_webhook_ca[count.index].cert_pem
+
+  validity_period_hours = 87600 # 10 years
+  early_renewal_hours   = 8760  # 1 year
+  allowed_uses = [
+    "key_encipherment",
+    "digital_signature"
+  ]
+}
+
 resource "helm_release" "aws_loadbalancer_controller" {
   count = local.aws_load_balancer_controller.enabled ? 1 : 0
 
@@ -331,5 +402,71 @@ resource "helm_release" "aws_loadbalancer_controller" {
   values = [
     local.aws_load_balancer_controller_values
   ]
+  set {
+    name  = "webhookTLS.caCert"
+    value = tls_self_signed_cert.aws_loadbalancer_controller_webhook_ca[0].cert_pem
+  }
+  set {
+    name  = "webhookTLS.cert"
+    value = tls_locally_signed_cert.aws_loadbalancer_controller_webhook[0].cert_pem
+  }
+  set {
+    name  = "webhookTLS.key"
+    value = tls_private_key.aws_loadbalancer_controller_webhook[0].private_key_pem
+  }
+}
+
+resource "kubernetes_ingress_v1" "default" {
+  count = local.aws_load_balancer_controller.enabled && local.ingress_nginx.enabled && var.nginx_ingress_ssl_terminator == "lb" ? 1 : 0
+
+  metadata {
+    name = "${local.ingress_nginx.name}-controller"
+    annotations = {
+      "kubernetes.io/ingress.class"                        = "alb"
+      "alb.ingress.kubernetes.io/scheme"                   = "internet-facing"
+      "alb.ingress.kubernetes.io/tags"                     = "Environment=${local.env},Name=${local.name},Cluster=${local.eks_cluster_id}"
+      "alb.ingress.kubernetes.io/certificate-arn"          = "${local.ssl_certificate_arn}"
+      "alb.ingress.kubernetes.io/ssl-policy"               = "ELBSecurityPolicy-FS-1-2-Res-2019-08"
+      "alb.ingress.kubernetes.io/listen-ports"             = "[{\"HTTP\": 80}, {\"HTTPS\": 443}]"
+      "alb.ingress.kubernetes.io/target-type"              = "ip"
+      "alb.ingress.kubernetes.io/load-balancer-attributes" = "routing.http2.enabled=true"
+      "alb.ingress.kubernetes.io/ssl-redirect"             = "443"
+    }
+    namespace = module.ingress_nginx_namespace[count.index].name
+  }
+  spec {
+    rule {
+      http {
+        path {
+          path = "/*"
+          backend {
+            service {
+              name = "${local.ingress_nginx.name}-controller"
+              port {
+                number = 80
+              }
+            }
+          }
+        }
+      }
+    }
+  }
+  wait_for_load_balancer = true
+
+  depends_on = [helm_release.aws_loadbalancer_controller, helm_release.ingress_nginx, module.aws_iam_aws_loadbalancer_controller, tls_locally_signed_cert.aws_loadbalancer_controller_webhook]
+}
+
+resource "aws_route53_record" "default_ingress" {
+  count = local.aws_load_balancer_controller.enabled && local.ingress_nginx.enabled && var.nginx_ingress_ssl_terminator == "lb" ? 1 : 0
 
+  zone_id = local.zone_id
+  name    = "*.${local.domain_name}"
+  type    = "CNAME"
+  ttl     = 360
+
+  records = [kubernetes_ingress_v1.default[count.index].status.0.load_balancer.0.ingress.0.hostname]
+
+  depends_on = [
+    kubernetes_ingress_v1.default
+  ]
 }
diff --git a/terraform/layer2-k8s/eks-ingress-nginx-controller.tf b/terraform/layer2-k8s/eks-ingress-nginx-controller.tf
@@ -7,8 +7,7 @@ locals {
     chart_version = local.helm_releases[index(local.helm_releases.*.id, "ingress-nginx")].chart_version
     namespace     = local.helm_releases[index(local.helm_releases.*.id, "ingress-nginx")].namespace
   }
-  ssl_certificate_arn                         = var.nginx_ingress_ssl_terminator == "lb" ? data.terraform_remote_state.layer1-aws.outputs.ssl_certificate_arn : "ssl-certificate"
-  ingress_nginx_general_values                = <<VALUES
+  ingress_nginx_general_values                   = <<VALUES
 rbac:
   create: true
 controller:
@@ -27,27 +26,20 @@ controller:
             values:
               - ON_DEMAND
 VALUES
-  ingress_loadbalancer_ssl_termination_values = <<VALUES
+  ingress_nginx_and_aws_load_balancer_controller = <<VALUES
 controller:
   service:
+    type: ClusterIP
     targetPorts:
       http: http
       https: http
-    annotations:
-      service.beta.kubernetes.io/aws-load-balancer-backend-protocol: http
-      service.beta.kubernetes.io/aws-load-balancer-connection-idle-timeout: '60'
-      service.beta.kubernetes.io/aws-load-balancer-ssl-ports: "https"
-      service.beta.kubernetes.io/aws-load-balancer-ssl-cert: ${local.ssl_certificate_arn}
-      service.beta.kubernetes.io/aws-load-balancer-ssl-negotiation-policy: ELBSecurityPolicy-TLS-1-2-2017-01
-      external-dns.alpha.kubernetes.io/hostname: ${local.domain_name}.
   publishService:
     enabled: true
   config:
     server-tokens: "false"
     use-forwarded-headers: "true"
-    set-real-ip-from: "${local.vpc_cidr}"
 VALUES
-  ingress_pod_ssl_termination_values          = <<VALUES
+  ingress_pod_ssl_termination_values             = <<VALUES
 controller:
   extraArgs:
     default-ssl-certificate: "${local.ingress_nginx.enabled ? module.ingress_nginx_namespace[0].name : "default"}/nginx-tls"
@@ -219,7 +211,7 @@ resource "helm_release" "ingress_nginx" {
 
   values = [
     local.ingress_nginx_general_values,
-    var.nginx_ingress_ssl_terminator == "lb" ? local.ingress_loadbalancer_ssl_termination_values : local.ingress_pod_ssl_termination_values
+    var.nginx_ingress_ssl_terminator == "lb" ? local.ingress_nginx_and_aws_load_balancer_controller : local.ingress_pod_ssl_termination_values
   ]
 
   depends_on = [kubectl_manifest.kube_prometheus_stack_operator_crds]
diff --git a/terraform/layer2-k8s/helm-releases.yaml b/terraform/layer2-k8s/helm-releases.yaml
@@ -1,6 +1,6 @@
 releases:
   - id: aws-load-balancer-controller
-    enabled: false
+    enabled: true
     chart: aws-load-balancer-controller
     repository: https://aws.github.io/eks-charts
     chart_version: 1.4.1
@@ -42,7 +42,7 @@ releases:
     chart_version:
     namespace: elk
   - id: external-dns
-    enabled: true
+    enabled: false
     chart: external-dns
     repository: https://kubernetes-sigs.github.io/external-dns
     chart_version: 1.9.0
