[CI] Add Windows Server 2022 Node Pools

This patch configures the windows server 2022 node pools and runner sets for
the transition from server 2019 to 2022.

Reviewers: cmtice, Keenuts, dschuff, gburgessiv, lnihlen

Reviewed By: cmtice

Pull Request: #498

Author: boomanaiden154

premerge/gke_cluster/main.tf

@@ -97,6 +97,8 @@ resource "google_container_node_pool" "llvm_premerge_libcxx" {
   }
 }
 
+# TODO(boomanaiden154): Make sure to delete this node pool after we have
+# switched over to server 2022.
 resource "google_container_node_pool" "llvm_premerge_windows" {
   name               = "llvm-premerge-windows"
   location           = var.region
@@ -138,3 +140,50 @@ resource "google_container_node_pool" "llvm_premerge_windows" {
     }
   }
 }
+
+resource "google_container_node_pool" "llvm_premerge_windows_2022" {
+  name               = "llvm-premerge-windows-2022"
+  location           = var.region
+  cluster            = google_container_cluster.llvm_premerge.name
+  initial_node_count = 0
+
+  # TODO(boomanaiden154): Bump this to full capacity (16 nodes) once we are
+  # ready to switch over to server 2022.
+  autoscaling {
+    total_min_node_count = 0
+    total_max_node_count = 2
+  }
+
+  # We do not set a taint for the windows nodes as kubernetes by default sets
+  # a node.kubernetes.io/os taint for windows nodes.
+  node_config {
+    machine_type = var.windows_machine_type
+    labels = {
+      "premerge-platform" : "windows-2022"
+    }
+    image_type = "WINDOWS_LTSC_CONTAINERD"
+    windows_node_config {
+      osversion = "OS_VERSION_LTSC2022"
+    }
+    # Add a script that runs on the initial boot to disable Windows Defender.
+    # Windows Defender causes an increase in test times by approximately an
+    # order of magnitude.
+    metadata = {
+      "sysprep-specialize-script-ps1" = "Set-MpPreference -DisableRealtimeMonitoring $true"
+      # Terraform wants to recreate the node pool everytime whe running
+      # terraform apply unless we explicitly set this.
+      # TODO(boomanaiden154): Look into why terraform is doing this so we do
+      # not need this hack.
+      "disable-legacy-endpoints" = "true"
+    }
+    disk_size_gb = 200
+    disk_type    = "pd-ssd"
+    # Terraform wants to recreate the node pool everytime whe running
+    # terraform apply unless we explicitly set this.
+    # TODO(boomanaiden154): Look into why terraform is doing this so we do
+    # not need this hack.
+    resource_labels = {
+      "goog-gke-node-pool-provisioning-model" = "on-demand"
+    }
+  }
+}

premerge/premerge_resources/main.tf

@@ -47,6 +47,12 @@ resource "kubernetes_namespace" "llvm_premerge_windows_runners" {
   }
 }
 
+resource "kubernetes_namespace" "llvm_premerge_windows_2022_runners" {
+  metadata {
+    name = "llvm-premerge-windows-2022-runners"
+  }
+}
+
 resource "kubernetes_secret" "linux_github_pat" {
   metadata {
     name      = "github-token"
@@ -132,6 +138,23 @@ resource "kubernetes_secret" "windows_github_pat" {
   depends_on = [kubernetes_namespace.llvm_premerge_windows_runners]
 }
 
+resource "kubernetes_secret" "windows_2022_github_pat" {
+  metadata {
+    name      = "github-token"
+    namespace = "llvm-premerge-windows-2022-runners"
+  }
+
+  data = {
+    "github_app_id"              = var.github_app_id
+    "github_app_installation_id" = var.github_app_installation_id
+    "github_app_private_key"     = var.github_app_private_key
+  }
+
+  type = "Opaque"
+
+  depends_on = [kubernetes_namespace.llvm_premerge_windows_2022_runners]
+}
+
 resource "helm_release" "github_actions_runner_controller" {
   name       = "llvm-premerge-controller"
   namespace  = "llvm-premerge-controller"
@@ -180,6 +203,24 @@ resource "helm_release" "github_actions_runner_set_windows" {
   ]
 }
 
+resource "helm_release" "github_actions_runner_set_windows_2022" {
+  name       = "llvm-premerge-windows-2022-runners"
+  namespace  = "llvm-premerge-windows-2022-runners"
+  repository = "oci://ghcr.io/actions/actions-runner-controller-charts"
+  version    = "0.11.0"
+  chart      = "gha-runner-scale-set"
+
+  values = [
+    "${templatefile("windows_2022_runner_values.yaml", { runner_group_name : var.runner_group_name })}"
+  ]
+
+  depends_on = [
+    kubernetes_namespace.llvm_premerge_windows_2022_runners,
+    kubernetes_secret.windows_2022_github_pat,
+    helm_release.github_actions_runner_controller,
+  ]
+}
+
 resource "helm_release" "github_actions_runner_set_libcxx" {
   name       = "llvm-premerge-libcxx-runners"
   namespace  = "llvm-premerge-libcxx-runners"

premerge/windows_2022_runner_values.yaml

@@ -0,0 +1,48 @@
+githubConfigUrl: "https://github.com/llvm"
+githubConfigSecret: "github-token"
+
+minRunners: 0
+maxRunners: 16
+
+runnerGroup: ${ runner_group_name }
+
+template:
+  metadata:
+    annotations:
+      cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+  spec:
+    tolerations:
+    - key: "node.kubernetes.io/os"
+      operator: "Equal"
+      value: "windows"
+      effect: "NoSchedule"
+    - key: "premerge-platform"
+      operator: "Equal"
+      value: "windows-2022"
+      effect: "NoSchedule"
+    nodeSelector:
+      premerge-platform: windows-2022
+    containers:
+      - name: runner
+        resources:
+          requests:
+            cpu: 28
+        image: ghcr.io/llvm/ci-windows-2022:latest
+        command: ["run.cmd"]
+        args: ["--jitconfig", "$(ACTIONS_RUNNER_INPUT_JITCONFIG)"]
+        env:
+          - name: DISABLE_RUNNER_UPDATE
+            value: "true"
+        # Add a volume/mount it to C:/_work so that we can use more than 20GB
+        # of space. Windows containers default to only having 20GB of scratch
+        # space and there is no way to configure this through kubernetes
+        # although the underlying containerd runtime should support it. So
+        # instead we just use a emptyDir volume mounted at C:/_work, which is
+        # where the GHA runner does everything.
+        volumeMounts:
+          - name: builddir
+            mountPath: "C:/_work"
+    volumes:
+      - name: builddir
+        emptyDir: {}
+