templates: ensure critical components dont coexist

This commit adds antiaffinity rules to the controller manager and the
sheduler to ensure that these deployments' pods do not all end up on the
same master node, especially now that they are pinned with node
selectors. To make this possible on small clusters, e.g. those with 1
master node, this is specified with "preferred" rather than "required".

Author: squat

pkg/asset/internal/templates.go

@@ -323,6 +323,22 @@ spec:
         tier: control-plane
         component: kube-controller-manager
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: tier
+                  operator: In
+                  values:
+                  - control-plane
+                - key: component
+                  operator: In
+                  values:
+                  - kube-contoller-manager
+              topologyKey: kubernetes.io/hostname
       nodeSelector:
         master: "true"
       containers:
@@ -388,6 +404,22 @@ spec:
         tier: control-plane
         component: kube-scheduler
     spec:
+      affinity:
+        podAntiAffinity:
+          preferredDuringSchedulingIgnoredDuringExecution:
+          - weight: 100
+            podAffinityTerm:
+              labelSelector:
+                matchExpressions:
+                - key: tier
+                  operator: In
+                  values:
+                  - control-plane
+                - key: component
+                  operator: In
+                  values:
+                  - kube-scheduler
+              topologyKey: kubernetes.io/hostname
       nodeSelector:
         master: "true"
       containers: