feat: each helm release has its own namespace (#164)

mglotov · web-flow · commit 18f106e5ad74 · 2021-11-05T17:57:36.000+06:00
diff --git a/docs/FAQ.md b/docs/FAQ.md
@@ -152,3 +152,60 @@ module "test_namespace" {
 }
 ```
 
+## How to add more restrictions for Gitlab-Runner
+By default Gitlab-Runner can deploy into any namespaces. If you want to allow Gitlab-Runner to deploy only into specific namespaces, then do these:
+* Create new Service Account:
+```
+resource "kubernetes_service_account" "gitlab_runner" {
+  metadata {
+    name      = "my-gitlab-runners-sa"
+    namespace = module.gitlab_runner_namespace.name
+    annotations = {
+      "eks.amazonaws.com/role-arn" = module.aws_iam_gitlab_runner.role_arn
+    }
+  }
+  automount_service_account_token = true
+}
+```
+* Create a new Kubernetes Role and RoleBinding. For example, these role and rolebinding will allow to deploy into dev namespace only:
+```
+resource "kubernetes_role" "dev" {
+  metadata {
+    name      = "${local.name}-dev"
+    namespace = "dev"
+  }
+
+  rule {
+    api_groups = ["", "apps", "extensions", "batch", "networking.k8s.io", "kubernetes-client.io"]
+    resources  = ["*"]
+    verbs      = ["*"]
+  }
+}
+
+resource "kubernetes_role_binding" "dev" {
+  metadata {
+    name      = "${local.name}-dev"
+    namespace = "dev"
+  }
+
+  role_ref {
+    api_group = "rbac.authorization.k8s.io"
+    kind      = "Role"
+    name      = kubernetes_role.dev.metadata.0.name
+  }
+
+  subject {
+    kind      = "ServiceAccount"
+    name      = kubernetes_service_account.gitlab_runner.metadata.0.name
+    namespace = module.gitlab_runner_namespace.name
+  }
+}
+```
+* Set the name of a new created account in layer2-k8s/templates/gitlab-runner-values.yaml
+```
+...
+runners:
+  serviceAccountName: my-gitlab-runners-sa
+  image: ubuntu:18.04
+...
+```
diff --git a/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf b/terraform/layer2-k8s/eks-aws-loadbalancer-controller.tf
@@ -13,14 +13,19 @@ locals {
   })
 }
 
+module "aws_load_balancer_controller_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "aws-load-balancer-controller"
+}
+
 resource "helm_release" "aws_loadbalancer_controller" {
   count = var.aws_loadbalancer_controller_enable ? 1 : 0
 
   name        = "aws-load-balancer-controller"
   chart       = local.aws-load-balancer-controller.chart
   repository  = local.aws-load-balancer-controller.repository
   version     = local.aws-load-balancer-controller.chart_version
-  namespace   = module.ing_namespace.name
+  namespace   = module.aws_load_balancer_controller_namespace.name
   max_history = var.helm_release_history_size
 
   values = [
diff --git a/terraform/layer2-k8s/eks-aws-node-termination-handler.tf b/terraform/layer2-k8s/eks-aws-node-termination-handler.tf
@@ -6,12 +6,17 @@ locals {
   }
 }
 
+module "aws_node_termination_handler_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "aws-node-termination-handler"
+}
+
 resource "helm_release" "aws_node_termination_handler" {
   name        = "aws-node-termination-handler"
   chart       = local.aws-node-termination-handler.chart
   repository  = local.aws-node-termination-handler.repository
   version     = local.aws-node-termination-handler.chart_version
-  namespace   = module.sys_namespace.name
+  namespace   = module.aws_node_termination_handler_namespace.name
   wait        = false
   max_history = var.helm_release_history_size
 
diff --git a/terraform/layer2-k8s/eks-cert-manager-certificate.tf b/terraform/layer2-k8s/eks-cert-manager-certificate.tf
@@ -20,7 +20,7 @@ resource "helm_release" "certificate" {
   chart       = local.cert-mananger-certificate.chart
   repository  = local.cert-mananger-certificate.repository
   version     = local.cert-mananger-certificate.chart_version
-  namespace   = module.ing_namespace.name
+  namespace   = module.ingress_nginx_namespace.name
   wait        = false
   max_history = var.helm_release_history_size
 
diff --git a/terraform/layer2-k8s/eks-cluster-autoscaler.tf b/terraform/layer2-k8s/eks-cluster-autoscaler.tf
@@ -17,12 +17,17 @@ data "template_file" "cluster_autoscaler" {
   }
 }
 
+module "cluster_autoscaler_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "cluster-autoscaler"
+}
+
 resource "helm_release" "cluster_autoscaler" {
   name        = "cluster-autoscaler"
   chart       = local.cluster-autoscaler.chart
   repository  = local.cluster-autoscaler.repository
   version     = local.cluster-autoscaler.chart_version
-  namespace   = module.sys_namespace.name
+  namespace   = module.cluster_autoscaler_namespace.name
   max_history = var.helm_release_history_size
 
   values = [
diff --git a/terraform/layer2-k8s/eks-external-dns.tf b/terraform/layer2-k8s/eks-external-dns.tf
@@ -16,12 +16,17 @@ data "template_file" "external_dns" {
   }
 }
 
+module "external_dns_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "external-dns"
+}
+
 resource "helm_release" "external_dns" {
   name        = "external-dns"
   chart       = local.external-dns.chart
   repository  = local.external-dns.repository
   version     = local.external-dns.chart_version
-  namespace   = module.dns_namespace.name
+  namespace   = module.external_dns_namespace.name
   max_history = var.helm_release_history_size
 
   values = [
diff --git a/terraform/layer2-k8s/eks-external-secrets.tf b/terraform/layer2-k8s/eks-external-secrets.tf
@@ -20,25 +20,35 @@ data "template_file" "external_secrets" {
   }
 }
 
+module "external_secrets_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "external-secrets"
+}
+
 resource "helm_release" "external_secrets" {
   name        = "external-secrets"
   chart       = local.external-secrets.chart
   repository  = local.external-secrets.repository
   version     = local.external-secrets.chart_version
-  namespace   = module.sys_namespace.name
+  namespace   = module.external_secrets_namespace.name
   max_history = var.helm_release_history_size
 
   values = [
     data.template_file.external_secrets.rendered,
   ]
 }
 
+module "reloader_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "reloader"
+}
+
 resource "helm_release" "reloader" {
   name        = "reloader"
   chart       = local.reloader.chart
   repository  = local.reloader.repository
   version     = local.reloader.chart_version
-  namespace   = module.sys_namespace.name
+  namespace   = module.reloader_namespace.name
   wait        = false
   max_history = var.helm_release_history_size
 }
diff --git a/terraform/layer2-k8s/eks-kube-prometheus-stack.tf b/terraform/layer2-k8s/eks-kube-prometheus-stack.tf
@@ -26,9 +26,9 @@ locals {
   })
 }
 
-resource "random_string" "grafana_password" {
-  length  = 20
-  special = true
+module "monitoring_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "monitoring"
 }
 
 resource "helm_release" "prometheus_operator" {
@@ -45,6 +45,11 @@ resource "helm_release" "prometheus_operator" {
   ]
 }
 
+resource "random_string" "grafana_password" {
+  length  = 20
+  special = true
+}
+
 module "aws_iam_grafana" {
   source = "../modules/aws-iam-eks-trusted"
 
diff --git a/terraform/layer2-k8s/eks-loki-stack.tf b/terraform/layer2-k8s/eks-loki-stack.tf
@@ -16,12 +16,17 @@ locals {
   })
 }
 
+module "loki_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "loki"
+}
+
 resource "helm_release" "loki_stack" {
   name        = "loki-stack"
   chart       = local.loki-stack.chart
   repository  = local.loki-stack.repository
   version     = local.loki-stack.chart_version
-  namespace   = module.monitoring_namespace.name
+  namespace   = module.loki_namespace.name
   wait        = false
   max_history = var.helm_release_history_size
 
diff --git a/terraform/layer2-k8s/eks-namespaces.tf b/terraform/layer2-k8s/eks-namespaces.tf
@@ -1,34 +1,4 @@
-module "dns_namespace" {
-  source = "../modules/kubernetes-namespace"
-  name   = "dns"
-}
-
-module "ing_namespace" {
-  source = "../modules/kubernetes-namespace"
-  name   = "ing"
-}
-
-module "elk_namespace" {
-  source = "../modules/kubernetes-namespace"
-  name   = "elk"
-}
-
 module "fargate_namespace" {
   source = "../modules/kubernetes-namespace"
   name   = "fargate"
 }
-
-module "ci_namespace" {
-  source = "../modules/kubernetes-namespace"
-  name   = "ci"
-}
-
-module "sys_namespace" {
-  source = "../modules/kubernetes-namespace"
-  name   = "sys"
-}
-
-module "monitoring_namespace" {
-  source = "../modules/kubernetes-namespace"
-  name   = "monitoring"
-}
diff --git a/terraform/layer2-k8s/eks-nginx-ingress-controller.tf b/terraform/layer2-k8s/eks-nginx-ingress-controller.tf
@@ -19,16 +19,21 @@ data "template_file" "nginx_ingress" {
     hostname           = local.domain_name
     ssl_cert           = local.ssl_certificate_arn
     proxy_real_ip_cidr = local.vpc_cidr
-    namespace          = module.ing_namespace.name
+    namespace          = module.ingress_nginx_namespace.name
   }
 }
 
-resource "helm_release" "nginx_ingress" {
+module "ingress_nginx_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "ingress-nginx"
+}
+
+resource "helm_release" "ingress_nginx" {
   name        = "ingress-nginx"
   chart       = local.ingress-nginx.chart
   repository  = local.ingress-nginx.repository
   version     = local.ingress-nginx.chart_version
-  namespace   = module.ing_namespace.name
+  namespace   = module.ingress_nginx_namespace.name
   wait        = false
   max_history = var.helm_release_history_size
 
diff --git a/terraform/layer2-k8s/examples/eks-elk.tf b/terraform/layer2-k8s/examples/eks-elk.tf
@@ -25,6 +25,11 @@ data "template_file" "elk" {
   }
 }
 
+module "elk_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "elk"
+}
+
 resource "helm_release" "elk" {
   name        = "elk"
   chart       = local.elk.chart
diff --git a/terraform/layer2-k8s/examples/eks-gitlab-runner.tf b/terraform/layer2-k8s/examples/eks-gitlab-runner.tf
@@ -11,27 +11,23 @@ locals {
       registration_token = local.gitlab_registration_token
       namespace          = module.ci_namespace.name
       role_arn           = module.aws_iam_gitlab_runner.role_arn
-      runner_sa          = module.eks_rbac_gitlab_runner.sa_name
       bucket_name        = local.gitlab_runner_cache_bucket_name
       region             = local.region
   })
 
 }
 
-module "eks_rbac_gitlab_runner" {
-  source = "../modules/eks-rbac-ci"
-
-  name      = "${local.name}-gl"
-  role_arn  = module.aws_iam_gitlab_runner.role_arn
-  namespace = module.ci_namespace.name
+module "gitlab_runner_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "gitlab-runner"
 }
 
 resource "helm_release" "gitlab_runner" {
   name        = "gitlab-runner"
   chart       = local.gitlab-runner.chart
   repository  = local.gitlab-runner.repository
   version     = local.gitlab-runner.chart_version
-  namespace   = module.ci_namespace.name
+  namespace   = module.gitlab_runner_namespace.name
   wait        = false
   max_history = var.helm_release_history_size
 
diff --git a/terraform/layer2-k8s/examples/eks-teamcity.tf b/terraform/layer2-k8s/examples/eks-teamcity.tf
@@ -23,6 +23,11 @@ data "template_file" "teamcity_agent" {
   }
 }
 
+module "teamcity_namespace" {
+  source = "../modules/kubernetes-namespace"
+  name   = "teamcity"
+}
+
 data "template_file" "teamcity" {
   template = file("${path.module}/templates/teamcity-values.yaml")
 
@@ -38,7 +43,7 @@ resource "helm_release" "teamcity" {
   chart           = local.teamcity.chart
   repository      = local.teamcity.repository
   version         = local.teamcity.chart_version
-  namespace       = module.ci_namespace.name
+  namespace       = module.teamcity_namespace.name
   wait            = false
   cleanup_on_fail = true
   max_history     = var.helm_release_history_size
diff --git a/terraform/layer2-k8s/templates/gitlab-runner-values.yaml b/terraform/layer2-k8s/templates/gitlab-runner-values.yaml
@@ -13,7 +13,6 @@ rbac:
     eks.amazonaws.com/role-arn: ${role_arn}
 
 runners:
-  serviceAccountName: ${runner_sa}
   image: ubuntu:18.04
   privileged: true
   namespace: ${namespace}
diff --git a/terraform/modules/eks-rbac-ci/main.tf b/terraform/modules/eks-rbac-ci/main.tf
diff --git a/terraform/modules/eks-rbac-ci/outputs.tf b/terraform/modules/eks-rbac-ci/outputs.tf
diff --git a/terraform/modules/eks-rbac-ci/variables.tf b/terraform/modules/eks-rbac-ci/variables.tf

Original file line number	Diff line number	Diff line change
`@@ -26,9 +26,9 @@ locals {`
`26`	`26`	`})`
`27`	`27`	`}`
`28`	`28`
`29`		`-resource "random_string" "grafana_password" {`
`30`		`- length = 20`
`31`		`- special = true`
	`29`	`+module "monitoring_namespace" {`
	`30`	`+ source = "../modules/kubernetes-namespace"`
	`31`	`+ name = "monitoring"`
`32`	`32`	`}`
`33`	`33`
`34`	`34`	`resource "helm_release" "prometheus_operator" {`
`@@ -45,6 +45,11 @@ resource "helm_release" "prometheus_operator" {`
`45`	`45`	`]`
`46`	`46`	`}`
`47`	`47`
	`48`	`+resource "random_string" "grafana_password" {`
	`49`	`+ length = 20`
	`50`	`+ special = true`
	`51`	`+}`
	`52`	`+`
`48`	`53`	`module "aws_iam_grafana" {`
`49`	`54`	`source = "../modules/aws-iam-eks-trusted"`
`50`	`55`