[CI] Add Terraform Plumbing for Postcommit CI

This patch does all the Terraform plumbing necessary for setting up the
postcommit buildbots. This leaves out the actual deployments for a future
patch.

Reviewers: Keenuts, lnihlen, gburgessiv, cmtice, dschuff

Reviewed By: cmtice

Pull Request: #542

Author: boomanaiden154

premerge/gke_cluster/main.tf

@@ -78,6 +78,40 @@ resource "google_container_node_pool" "llvm_premerge_linux" {
   }
 }
 
+# Buildbot here refers specifically to the LLVM Buildbot postcommit
+# testing infrastructure. These machines are used specifically for testing
+# commits after they have landed in main.
+resource "google_container_node_pool" "llvm_buildbot_linux" {
+  name               = "llvm-buildbot-linux"
+  location           = var.region
+  cluster            = google_container_cluster.llvm_premerge.name
+  initial_node_count = 0
+
+  autoscaling {
+    total_min_node_count = 0
+    total_max_node_count = 3
+  }
+
+  node_config {
+    machine_type = var.linux_machine_type
+    taint {
+      key    = "buildbot-platform"
+      value  = "linux"
+      effect = "NO_SCHEDULE"
+    }
+    labels = {
+      "buildbot-platform" : "linux"
+    }
+    disk_size_gb = 200
+
+    # Enable workload identity federation for this pool so that we can access
+    # GCS buckets.
+    workload_metadata_config {
+      mode = "GKE_METADATA"
+    }
+  }
+}
+
 resource "google_container_node_pool" "llvm_premerge_libcxx" {
   name               = "llvm-premerge-libcxx"
   location           = var.region
@@ -118,6 +152,56 @@ resource "google_container_node_pool" "llvm_premerge_windows_2022" {
   # a node.kubernetes.io/os taint for windows nodes.
   node_config {
     machine_type = var.windows_machine_type
+    labels = {
+      "buildbot-platform" : "windows-2022"
+    }
+    image_type = "WINDOWS_LTSC_CONTAINERD"
+    windows_node_config {
+      osversion = "OS_VERSION_LTSC2022"
+    }
+    # Add a script that runs on the initial boot to disable Windows Defender.
+    # Windows Defender causes an increase in test times by approximately an
+    # order of magnitude.
+    metadata = {
+      "sysprep-specialize-script-ps1" = "Set-MpPreference -DisableRealtimeMonitoring $true"
+      # Terraform wants to recreate the node pool everytime whe running
+      # terraform apply unless we explicitly set this.
+      # TODO(boomanaiden154): Look into why terraform is doing this so we do
+      # not need this hack.
+      "disable-legacy-endpoints" = "true"
+    }
+    disk_size_gb = 200
+    disk_type    = "pd-ssd"
+
+    # Enable workload identity federation for this pool so that we can access
+    # GCS buckets.
+    workload_metadata_config {
+      mode = "GKE_METADATA"
+    }
+  }
+}
+
+# Buildbot here refers specifically to the LLVM Buildbot postcommit
+# testing infrastructure. These machines are used specifically for testing
+# commits after they have landed in main.
+resource "google_container_node_pool" "llvm_buildbot_window_2022" {
+  name               = "llvm-buildbot-windows-2022"
+  location           = var.region
+  cluster            = google_container_cluster.llvm_premerge.name
+  initial_node_count = 0
+
+  autoscaling {
+    total_min_node_count = 0
+    total_max_node_count = 3
+  }
+
+  # We do not set a taint for the windows nodes as kubernetes by default sets
+  # a node.kubernetes.io/os taint for windows nodes.
+  node_config {
+    # Use the Linux machine type here as we want to keep the windows machines
+    # symmetric with the Linux machines for faster builds. Throughput is not
+    # as much of a concern postcommit.
+    machine_type = var.linux_machine_type
     labels = {
       "premerge-platform" : "windows-2022"
     }

premerge/main.tf

@@ -121,6 +121,25 @@ data "google_secret_manager_secret_version" "grafana_token" {
   secret = "llvm-premerge-testing-grafana-token"
 }
 
+# Buildbot here refers specifically to the LLVM Buildbot postcommit
+# testing infrastructure. These machines are used specifically for testing
+# commits after they have landed in main.
+data "google_secret_manager_secret_version" "us_central_linux_buildbot_password" {
+  secret = "llvm-buildbot-linux-us-central"
+}
+
+data "google_secret_manager_secret_version" "us_central_windows_buildbot_password" {
+  secret = "llvm-buildbot-windows-us-central"
+}
+
+data "google_secret_manager_secret_version" "us_west_linux_buildbot_password" {
+  secret = "llvm-buildbot-linux-us-west"
+}
+
+data "google_secret_manager_secret_version" "us_west_windows_buildbot_password" {
+  secret = "llvm-buildbot-windows-us-west"
+}
+
 provider "kubernetes" {
   host  = "https://${module.premerge_cluster_us_central.endpoint}"
   token = data.google_client_config.current.access_token
@@ -152,6 +171,10 @@ module "premerge_cluster_us_central_resources" {
   linux_object_cache_gcp_service_account_email         = module.premerge_cluster_us_central.linux_object_cache_gcp_service_account_email
   windows_2022_object_cache_gcp_service_account_email  = module.premerge_cluster_us_central.windows_2022_object_cache_gcp_service_account_email
   github_arc_version                                   = "0.12.1"
+  linux_buildbot_name                                  = "premerge-us-central-linux"
+  linux_buildbot_password                              = data.google_secret_manager_secret_version.us_central_linux_buildbot_password.secret_data
+  windows_buildbot_name                                = "premerge-us-central-windows"
+  windows_buildbot_password                            = data.google_secret_manager_secret_version.us_central_windows_buildbot_password.secret_data
   providers = {
     kubernetes = kubernetes.llvm-premerge-us-central
     helm       = helm.llvm-premerge-us-central
@@ -173,6 +196,10 @@ module "premerge_cluster_us_west_resources" {
   linux_object_cache_gcp_service_account_email         = module.premerge_cluster_us_west.linux_object_cache_gcp_service_account_email
   windows_2022_object_cache_gcp_service_account_email  = module.premerge_cluster_us_west.windows_2022_object_cache_gcp_service_account_email
   github_arc_version                                   = "0.12.1"
+  linux_buildbot_name                                  = "premerge-us-west-linux"
+  linux_buildbot_password                              = data.google_secret_manager_secret_version.us_west_linux_buildbot_password.secret_data
+  windows_buildbot_name                                = "premerge-us-west-windows"
+  windows_buildbot_password                            = data.google_secret_manager_secret_version.us_west_windows_buildbot_password.secret_data
   providers = {
     kubernetes = kubernetes.llvm-premerge-us-west
     helm       = helm.llvm-premerge-us-west

premerge/premerge_resources/main.tf

@@ -47,6 +47,51 @@ resource "kubernetes_namespace" "llvm_premerge_windows_2022_runners" {
   }
 }
 
+# Buildbot here refers specifically to the LLVM Buildbot postcommit
+# testing infrastructure. These machines are used specifically for testing
+# commits after they have landed in main.
+resource "kubernetes_namespace" "llvm_premerge_linux_buildbot" {
+  metadata {
+    name = "llvm-premerge-linux-buildbot"
+  }
+}
+
+resource "kubernetes_namespace" "llvm_premerge_windows_2022_buildbot" {
+  metadata {
+    name = "llvm-premerge-windows-2022-buildbot"
+  }
+}
+
+resource "kubernetes_secret" "linux_buildbot_password" {
+  metadata {
+    name      = "linux-buildbot-password"
+    namespace = "llvm-premerge-linux-buildbot"
+  }
+
+  data = {
+    "password" = var.linux_buildbot_password
+  }
+
+  type = "Opaque"
+
+  depends_on = [kubernetes_namespace.llvm_premerge_linux_buildbot]
+}
+
+resource "kubernetes_secret" "windows_2022_buildbot_password" {
+  metadata {
+    name      = "windows-buildbot-password"
+    namespace = "llvm-premerge-windows-buildbot"
+  }
+
+  data = {
+    "password" = var.windows_buildbot_password
+  }
+
+  type = "Opaque"
+
+  depends_on = [kubernetes_namespace.llvm_premerge_windows_2022_buildbot]
+}
+
 resource "kubernetes_secret" "linux_github_pat" {
   metadata {
     name      = "github-token"

premerge/premerge_resources/variables.tf

@@ -104,3 +104,23 @@ variable "windows_2022_object_cache_gcp_service_account_email" {
   description = "The email associated with the service account for accessing the object cache on Windows."
   type        = string
 }
+
+variable "linux_buildbot_name" {
+  description = "The name of the linux buildbot that will run tests postcommit."
+  type        = string
+}
+
+variable "linux_buildbot_password" {
+  description = "The password for the linux buildbot that will run tests postcommit."
+  type        = string
+}
+
+variable "windows_buildbot_name" {
+  description = "The name of the windows buildbot that will run tests postcommit."
+  type        = string
+}
+
+variable "windows_buildbot_password" {
+  description = "The password for the windows buildbot that will run tests postcommit."
+  type        = string
+}