[CI] Refactor helm charts and monitoring into module

This patch refactors the helm charts and monitoring into a terraform module.
This makes it easier to reuse them for a HA setup where we have multiple
clusters.

We leave the metrics container out on purpose as we only need to run it in
one cluster.

Reviewers: cmtice, dschuff, Keenuts, gburgessiv, lnihlen

Reviewed By: lnihlen

Pull Request: #442

Author: boomanaiden154

premerge/main.tf

@@ -79,6 +79,7 @@ provider "helm" {
     client_key             = base64decode(module.premerge_cluster.client_key)
     cluster_ca_certificate = base64decode(module.premerge_cluster.cluster_ca_certificate)
   }
+  alias = "llvm-premerge-prototype"
 }
 
 data "google_secret_manager_secret_version" "github_app_id" {
@@ -103,183 +104,72 @@ provider "kubernetes" {
   cluster_ca_certificate = base64decode(
     module.premerge_cluster.cluster_ca_certificate
   )
+  alias = "llvm-premerge-prototype"
 }
 
-resource "kubernetes_namespace" "llvm_premerge_controller" {
-  metadata {
-    name = "llvm-premerge-controller"
+module "premerge_cluster_resources" {
+  source                     = "./premerge_resources"
+  github_app_id              = data.google_secret_manager_secret_version.github_app_id.secret_data
+  github_app_installation_id = data.google_secret_manager_secret_version.github_app_installation_id.secret_data
+  github_app_private_key     = data.google_secret_manager_secret_version.github_app_private_key.secret_data
+  cluster_name               = "llvm-premerge-prototype"
+  grafana_token              = data.google_secret_manager_secret_version.grafana_token.secret_data
+  providers = {
+    kubernetes = kubernetes.llvm-premerge-prototype
+    helm       = helm.llvm-premerge-prototype
   }
 }
 
-resource "kubernetes_namespace" "llvm_premerge_linux_runners" {
-  metadata {
-    name = "llvm-premerge-linux-runners"
-  }
+# TODO(boomanaiden154): Remove these moved blocks once we have finished
+# updating everything to use the new module.
+moved {
+  from = kubernetes_namespace.llvm_premerge_controller
+  to   = module.premerge_cluster_resources.kubernetes_namespace.llvm_premerge_controller
 }
 
-resource "kubernetes_secret" "linux_github_pat" {
-  metadata {
-    name      = "github-token"
-    namespace = "llvm-premerge-linux-runners"
-  }
-
-  data = {
-    "github_app_id"              = data.google_secret_manager_secret_version.github_app_id.secret_data
-    "github_app_installation_id" = data.google_secret_manager_secret_version.github_app_installation_id.secret_data
-    "github_app_private_key"     = data.google_secret_manager_secret_version.github_app_private_key.secret_data
-  }
-
-  type = "Opaque"
+moved {
+  from = kubernetes_namespace.llvm_premerge_linux_runners
+  to   = module.premerge_cluster_resources.kubernetes_namespace.llvm_premerge_linux_runners
 }
 
-resource "kubernetes_namespace" "llvm_premerge_windows_runners" {
-  metadata {
-    name = "llvm-premerge-windows-runners"
-  }
+moved {
+  from = kubernetes_secret.linux_github_pat
+  to   = module.premerge_cluster_resources.kubernetes_secret.linux_github_pat
 }
 
-resource "kubernetes_secret" "windows_github_pat" {
-  metadata {
-    name      = "github-token"
-    namespace = "llvm-premerge-windows-runners"
-  }
-
-  data = {
-    "github_app_id"              = data.google_secret_manager_secret_version.github_app_id.secret_data
-    "github_app_installation_id" = data.google_secret_manager_secret_version.github_app_installation_id.secret_data
-    "github_app_private_key"     = data.google_secret_manager_secret_version.github_app_private_key.secret_data
-  }
-
-  type = "Opaque"
+moved {
+  from = kubernetes_namespace.llvm_premerge_windows_runners
+  to   = module.premerge_cluster_resources.kubernetes_namespace.llvm_premerge_windows_runners
 }
 
-
-resource "helm_release" "github_actions_runner_controller" {
-  name       = "llvm-premerge-controller"
-  namespace  = "llvm-premerge-controller"
-  repository = "oci://ghcr.io/actions/actions-runner-controller-charts"
-  version    = "0.11.0"
-  chart      = "gha-runner-scale-set-controller"
-
-  depends_on = [
-    kubernetes_namespace.llvm_premerge_controller
-  ]
+moved {
+  from = kubernetes_secret.windows_github_pat
+  to   = module.premerge_cluster_resources.kubernetes_secret.windows_github_pat
 }
 
-resource "helm_release" "github_actions_runner_set_linux" {
-  name       = "llvm-premerge-linux-runners"
-  namespace  = "llvm-premerge-linux-runners"
-  repository = "oci://ghcr.io/actions/actions-runner-controller-charts"
-  version    = "0.11.0"
-  chart      = "gha-runner-scale-set"
-
-  values = [
-    "${file("linux_runners_values.yaml")}"
-  ]
-
-  depends_on = [
-    kubernetes_namespace.llvm_premerge_linux_runners,
-    helm_release.github_actions_runner_controller,
-    kubernetes_secret.linux_github_pat,
-  ]
+moved {
+  from = helm_release.github_actions_runner_controller
+  to   = module.premerge_cluster_resources.helm_release.github_actions_runner_controller
 }
 
-resource "helm_release" "github_actions_runner_set_windows" {
-  name       = "llvm-premerge-windows-runners"
-  namespace  = "llvm-premerge-windows-runners"
-  repository = "oci://ghcr.io/actions/actions-runner-controller-charts"
-  version    = "0.11.0"
-  chart      = "gha-runner-scale-set"
-
-  values = [
-    "${file("windows_runner_values.yaml")}"
-  ]
-
-  depends_on = [
-    kubernetes_namespace.llvm_premerge_windows_runners,
-    kubernetes_secret.windows_github_pat,
-    helm_release.github_actions_runner_controller,
-  ]
+moved {
+  from = helm_release.github_actions_runner_set_linux
+  to   = module.premerge_cluster_resources.helm_release.github_actions_runner_set_linux
 }
 
-resource "kubernetes_namespace" "grafana" {
-  metadata {
-    name = "grafana"
-  }
+moved {
+  from = helm_release.github_actions_runner_set_windows
+  to   = module.premerge_cluster_resources.helm_release.github_actions_runner_set_windows
 }
 
-resource "helm_release" "grafana-k8s-monitoring" {
-  name             = "grafana-k8s-monitoring"
-  repository       = "https://grafana.github.io/helm-charts"
-  chart            = "k8s-monitoring"
-  namespace        = "grafana"
-  create_namespace = true
-  atomic           = true
-  timeout          = 300
-
-  values = [file("${path.module}/grafana_values.yaml")]
-
-  set {
-    name  = "cluster.name"
-    value = var.cluster_name
-  }
-
-  set {
-    name  = "externalServices.prometheus.host"
-    value = var.externalservices_prometheus_host
-  }
-
-  set_sensitive {
-    name  = "externalServices.prometheus.basicAuth.username"
-    value = var.externalservices_prometheus_basicauth_username
-  }
-
-  set_sensitive {
-    name  = "externalServices.prometheus.basicAuth.password"
-    value = data.google_secret_manager_secret_version.grafana_token.secret_data
-  }
-
-  set {
-    name  = "externalServices.loki.host"
-    value = var.externalservices_loki_host
-  }
-
-  set_sensitive {
-    name  = "externalServices.loki.basicAuth.username"
-    value = var.externalservices_loki_basicauth_username
-  }
-
-  set_sensitive {
-    name  = "externalServices.loki.basicAuth.password"
-    value = data.google_secret_manager_secret_version.grafana_token.secret_data
-  }
-
-  set {
-    name  = "externalServices.tempo.host"
-    value = var.externalservices_tempo_host
-  }
-
-  set_sensitive {
-    name  = "externalServices.tempo.basicAuth.username"
-    value = var.externalservices_tempo_basicauth_username
-  }
-
-  set_sensitive {
-    name  = "externalServices.tempo.basicAuth.password"
-    value = data.google_secret_manager_secret_version.grafana_token.secret_data
-  }
-
-  set {
-    name  = "opencost.opencost.exporter.defaultClusterId"
-    value = var.cluster_name
-  }
-
-  set {
-    name  = "opencost.opencost.prometheus.external.url"
-    value = format("%s/api/prom", var.externalservices_prometheus_host)
-  }
+moved {
+  from = kubernetes_namespace.grafana
+  to   = module.premerge_cluster_resources.kubernetes_namespace.grafana
+}
 
-  depends_on = [kubernetes_namespace.grafana]
+moved {
+  from = helm_release.grafana-k8s-monitoring
+  to   = module.premerge_cluster_resources.helm_release.grafana-k8s-monitoring
 }
 
 data "google_secret_manager_secret_version" "metrics_github_pat" {
@@ -298,11 +188,11 @@ data "google_secret_manager_secret_version" "metrics_buildkite_token" {
   secret = "llvm-premerge-metrics-buildkite-graphql-token"
 }
 
-
 resource "kubernetes_namespace" "metrics" {
   metadata {
     name = "metrics"
   }
+  provider = kubernetes.llvm-premerge-prototype
 }
 
 resource "kubernetes_secret" "metrics_secrets" {
@@ -318,9 +208,11 @@ resource "kubernetes_secret" "metrics_secrets" {
     "buildkite-token"        = data.google_secret_manager_secret_version.metrics_buildkite_token.secret_data
   }
 
-  type = "Opaque"
+  type     = "Opaque"
+  provider = kubernetes.llvm-premerge-prototype
 }
 
 resource "kubernetes_manifest" "metrics_deployment" {
   manifest = yamldecode(file("metrics_deployment.yaml"))
+  provider = kubernetes.llvm-premerge-prototype
 }