Use scheduling controls for skipper-ingress-redis

Signed-off-by: Mikkel Oscar Lyderik Larsen <[email protected]>

Author: mikkeloscar

cluster/manifests/skipper/skipper-redis.yaml

@@ -22,6 +22,7 @@ spec:
         version: v7.2.4
       annotations:
         cluster-autoscaler.kubernetes.io/safe-to-evict: "false"
+        karpenter.sh/do-not-disrupt: "true"
         logging/destination: "{{ .Cluster.ConfigItems.log_destination_both }}"
 {{- if eq .Cluster.ConfigItems.skipper_topology_spread_enabled "true" }}
         zalando.org/topology-spread-timeout: 7m
@@ -30,6 +31,7 @@ spec:
 {{- if eq .Cluster.ConfigItems.skipper_topology_spread_enabled "true" }}
       topologySpreadConstraints:
         - maxSkew: 1
+          minDomains: 3
           topologyKey: topology.kubernetes.io/zone
           whenUnsatisfiable: DoNotSchedule
           labelSelector:
@@ -47,13 +49,6 @@ spec:
                 values:
                 - skipper-ingress-redis
         nodeAffinity:
-          requiredDuringSchedulingIgnoredDuringExecution:
-            nodeSelectorTerms:
-            - matchExpressions:
-              - key: node.kubernetes.io/node-pool
-                operator: In
-                values:
-                - skipper-ingress-redis
           preferredDuringSchedulingIgnoredDuringExecution:
           - weight: 100
             preference:
@@ -106,9 +101,9 @@ spec:
       schedulerName: default-scheduler
 {{ if eq .Cluster.ConfigItems.enable_dedicate_nodepool_skipper_redis "true"}}
       nodeSelector:
-        dedicated: skipper-ingress-redis
+        zalando.org/dedicated: skipper-ingress-redis
       tolerations:
-      - effect: NoSchedule
-        key: dedicated
-        value: skipper-ingress-redis
+      - key: "zalando.org/dedicated"
+        operator: Exists
+        effect: NoSchedule
 {{ end }}

test/e2e/cluster_config.sh

@@ -164,7 +164,7 @@ EOFF
     - "g6.xlarge"
     - "g6.2xlarge"
     - "g6.4xlarge"
-    name: karpenter-gpu
+    name: karpenter-gpu-tesla
     profile: worker-karpenter
     min_size: 0
     max_size: 0
@@ -181,6 +181,52 @@ EOFF
     config_items:
       labels: dedicated=node-reboot-tests
       taints: dedicated=node-reboot-tests:NoSchedule
+  - config_items:
+      requirements: "- key: karpenter.k8s.aws/instance-gpu-manufacturer\n  operator: In\n  values:\n  - nvidia\n- key: zalando.org/dedicated\n  operator: Exists\n"
+      scaling_priority: "2"
+      taints: nvidia.com/gpu=present:NoSchedule,zalando.org/dedicated=dedicated:NoSchedule
+    discount_strategy: none
+    instance_type: not-specified
+    instance_types:
+    - not-specified
+    max_size: 0
+    min_size: 0
+    name: karpenter-gpu-dedicated
+    profile: worker-karpenter
+  - config_items:
+      requirements: "- key: zalando.org/dedicated\n  operator: Exists\n"
+      scaling_priority: "1"
+      taints: zalando.org/dedicated=dedicated:NoSchedule
+    discount_strategy: none
+    instance_type: not-specified
+    instance_types:
+    - not-specified
+    max_size: 0
+    min_size: 0
+    name: karpenter-catch-all-dedicated
+    profile: worker-karpenter
+  - config_items:
+      requirements: "- key: karpenter.k8s.aws/instance-gpu-manufacturer\n  operator: In\n  values:\n  - nvidia\n"
+      scaling_priority: "3"
+      taints: nvidia.com/gpu=present:NoSchedule
+    discount_strategy: none
+    instance_type: not-specified
+    instance_types:
+    - not-specified
+    max_size: 0
+    min_size: 0
+    name: karpenter-gpu
+    profile: worker-karpenter
+  - config_items:
+      scaling_priority: "2"
+    discount_strategy: none
+    instance_type: not-specified
+    instance_types:
+    - not-specified
+    max_size: 0
+    min_size: 0
+    name: karpenter-catch-all
+    profile: worker-karpenter
   provider: ${CLUSTER_PROVIDER}
   region: ${REGION}
   owner: '${OWNER}'