[CI] Use Grace Periods instead of PDBs

GKE apparently does not like to upgrade the cluster with how we were attempting
to use PDBs. Switch over to grace periods specified per pod, which is probably
closer to what we actually intend anyways. This might have the chance of killing
a still running job, but that should be pretty rare given the vast majority of
jobs complete in under an hour and that we have maintenance windows set to
only run jobs in the deep troughs of demand.

Author: boomanaiden154

premerge/libcxx_runners_values.yaml

@@ -11,6 +11,7 @@ template:
     annotations:
       cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
   spec:
+    terminationGracePeriodSeconds: 3600
     tolerations:
     - key: "premerge-platform-libcxx"
       operator: "Equal"

premerge/linux_runners_values.yaml

@@ -12,6 +12,7 @@ template:
       cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
   spec:
     serviceAccountName: linux-runners-ksa
+    terminationGracePeriodSeconds: 3600
     tolerations:
     - key: "premerge-platform"
       operator: "Equal"

premerge/pod_disruption_budget.yaml

@@ -458,36 +458,6 @@ resource "kubernetes_service_account" "windows_2022_object_cache_ksa" {
   depends_on = [kubernetes_namespace.llvm_premerge_windows_2022_runners]
 }
 
-# We set up pod disruption budgets here. We need one per namespace and we need
-# to set the min pod count to the maximum number of runner pods that can
-# possibly exist so we never have a number of disruptible pods greater than
-# zero.
-
-resource "kubernetes_manifest" "linux_runners_disruption_budget" {
-  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-linux-runners", min_pod_count : 16 }))
-  depends_on = [kubernetes_namespace.llvm_premerge_linux_runners]
-}
-
-resource "kubernetes_manifest" "windows_2022_runners_disruption_budget" {
-  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-windows-2022-runners", min_pod_count : 16 }))
-  depends_on = [kubernetes_namespace.llvm_premerge_windows_2022_runners]
-}
-
-resource "kubernetes_manifest" "libcxx_runners_disruption_budget" {
-  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-libcxx-runners", min_pod_count : 32 }))
-  depends_on = [kubernetes_namespace.llvm_premerge_libcxx_runners]
-}
-
-resource "kubernetes_manifest" "libcxx_release_runners_disruption_budget" {
-  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-libcxx-release-runners", min_pod_count : 32 }))
-  depends_on = [kubernetes_namespace.llvm_premerge_libcxx_release_runners]
-}
-
-resource "kubernetes_manifest" "libcxx_next_runners_disruption_budget" {
-  manifest   = yamldecode(templatefile("pod_disruption_budget.yaml", { runner_set_name : "llvm-premerge-libcxx-next-runners", min_pod_count : 32 }))
-  depends_on = [kubernetes_namespace.llvm_premerge_libcxx_next_runners]
-}
-
 resource "kubernetes_namespace" "premerge_advisor" {
   metadata {
     name = "premerge-advisor"

premerge/premerge_resources/main.tf

@@ -12,6 +12,7 @@ template:
       cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
   spec:
     serviceAccountName: windows-runners-ksa
+    terminationGracePeriodSeconds: 3600
     tolerations:
     - key: "node.kubernetes.io/os"
       operator: "Equal"