Merge pull request #8915 from zalando-incubator/register-with-taints-kubelet-config

Move registerWithTaint to userdata.yaml

Author: mikkeloscar

cluster/config-defaults.yaml

@@ -756,12 +756,14 @@ tracing_coredns_local_zone_traces_endpoint: ""
 # AMI id given the image name and the Image AWS account owner.
 #
 # [0]: https://github.com/zalando-incubator/cluster-lifecycle-manager/blob/8a9bd1cb2d094038a9e23e646421f8146b48886a/provisioner/template.go#L116
-kuberuntu_image_v1_31_jammy_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.4-amd64-master-359" "861068367966" }}
-kuberuntu_image_v1_31_jammy_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.4-arm64-master-359" "861068367966" }}
-
-# Which distro from the previous config items should be used. Valid options are only `jammy` for now. Can be set for each node pool.
-kuberuntu_distro_master: "jammy"
-kuberuntu_distro_worker: "jammy"
+kuberuntu_image_v1_31_old_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.4-amd64-master-359" "861068367966" }}
+kuberuntu_image_v1_31_old_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.4-arm64-master-359" "861068367966" }}
+kuberuntu_image_v1_31_new_amd64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.6-amd64-master-368" "861068367966" }}
+kuberuntu_image_v1_31_new_arm64: {{ amiID "zalando-ubuntu-jammy-22.04-kubernetes-production-v1.31.6-arm64-master-368" "861068367966" }}
+
+# This is used to determine which AMI to use for the cluster or individual node
+# pools. Possible values are 'new' or 'old'
+kuberuntu_ami_version: "new"
 
 # Feature toggle for auditing events
 audit_pod_events: "true"

cluster/node-pools/master-default/stack.yaml

@@ -10,7 +10,7 @@ Mappings:
   Images:
     {{.Cluster.Region}}:
       # Use the node pool's architecture to construct the config item name that we're using to get the AMI name.
-      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_master "_" .Values.InstanceInfo.Architecture) }}'
+      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_ami_version "_" .Values.InstanceInfo.Architecture) }}'
 
 Resources:
   AutoScalingGroup:

cluster/node-pools/master-default/userdata.yaml

@@ -3,7 +3,9 @@ write_files:
   - owner: root:root
     path: /etc/kubernetes/secrets.env
     content: |
+{{- if eq .NodePool.ConfigItems.kuberuntu_ami_version "old" }}
       NODEPOOL_TAINTS=node.kubernetes.io/role=master:NoSchedule{{if index .NodePool.ConfigItems "taints"}},{{.NodePool.ConfigItems.taints}}{{end}}
+{{- end }}
       NODE_LABELS=master=true,node.kubernetes.io/exclude-from-external-load-balancers,node.kubernetes.io/distro=ubuntu,cluster-lifecycle-controller.zalan.do/decommission-priority=999,lifecycle-status=ready{{if index .NodePool.ConfigItems "labels"}},{{.NodePool.ConfigItems.labels}}{{end}}
       NODEPOOL_NAME={{ .NodePool.Name }}
       KUBELET_ROLE=master
@@ -76,6 +78,19 @@ write_files:
       # variables are replaced on instance start-up.
       providerID: __PROVIDER_ID__
       clusterDNS: [__CLUSTER_DNS__]
+{{- if eq .NodePool.ConfigItems.kuberuntu_ami_version "new" }}
+      registerWithTaints:
+        - key: node.kubernetes.io/role
+          value: master
+          effect: NoSchedule
+{{- range $taint := .NodePool.Taints }}
+        - key: {{$taint.Key}}
+          effect: {{$taint.Effect}}
+{{- if $taint.Value }}
+          value: {{$taint.Value }}
+{{- end }}
+{{- end }}
+{{- end }}
 
   - owner: root:root
     path: /etc/kubernetes/manifests/kube-apiserver.yaml
@@ -923,7 +938,7 @@ write_files:
       {{index $cfg 0}} = {{index $cfg 1}}
       {{- end}}
 {{- end}}
-
+{{ if eq .NodePool.ConfigItems.kuberuntu_ami_version "old" }}
   # TODO: Remove this once all nodes are running an AMI compatible with /etc/cni/net.d/10-flannel.conflist
   - owner: root:root
     path: /etc/kubernetes/cni/net.d/10-flannel.conflist
@@ -941,7 +956,7 @@ write_files:
           }
         ]
       }
-
+{{ end }}
   - owner: root:root
     path: /etc/cni/net.d/10-flannel.conflist
     content: |

cluster/node-pools/worker-combined/stack.yaml

@@ -10,7 +10,7 @@ Mappings:
   Images:
     {{.Cluster.Region}}:
       # Use the node pool's architecture to construct the config item name that we're using to get the AMI name.
-      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
+      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_ami_version "_" .Values.InstanceInfo.Architecture) }}'
 
 Resources:
   AutoScalingGroup:

cluster/node-pools/worker-karpenter/provisioners.yaml

@@ -8,8 +8,8 @@ spec:
   amiFamily: Custom
   amiSelectorTerms:
     # Select on any AMI that has any of the following IDs
-    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_worker "_amd64") }}
-    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_worker "_arm64") }}
+    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_ami_version "_amd64") }}
+    - id: {{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_ami_version "_arm64") }}
   metadataOptions:
     httpEndpoint: enabled
     httpProtocolIPv6: disabled

cluster/node-pools/worker-splitaz/stack.yaml

@@ -10,7 +10,7 @@ Mappings:
   Images:
     {{.Cluster.Region}}:
       # Use the node pool's architecture to construct the config item name that we're using to get the AMI name.
-      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_distro_worker "_" .Values.InstanceInfo.Architecture) }}'
+      MachineImage: '{{ index .NodePool.ConfigItems (print "kuberuntu_image_v1_31_" .NodePool.ConfigItems.kuberuntu_ami_version "_" .Values.InstanceInfo.Architecture) }}'
 
 Resources:
 {{ with $data := . }}

cluster/node-pools/worker-splitaz/userdata.yaml

@@ -7,7 +7,9 @@ write_files:
   - owner: root:root
     path: /etc/kubernetes/secrets.env
     content: |
+{{- if eq .NodePool.ConfigItems.kuberuntu_ami_version "old" }}
       NODEPOOL_TAINTS={{if index .NodePool.ConfigItems "taints"}}{{.NodePool.ConfigItems.taints}}{{end}}
+{{- end }}
       NODE_LABELS=node.kubernetes.io/profile={{ .NodePool.Profile }},lifecycle-status=ready,node.kubernetes.io/distro=ubuntu{{if index .NodePool.ConfigItems "labels"}},{{.NodePool.ConfigItems.labels}}{{end}}{{if or (eq .NodePool.Profile "worker-splitaz") (eq .NodePool.Profile "worker-combined")}},asg-lifecycle-hook=true{{end}}
       NODEPOOL_NAME={{ .NodePool.Name }}
       KUBELET_ROLE=worker
@@ -118,6 +120,22 @@ write_files:
       # variables are replaced on instance start-up.
       providerID: __PROVIDER_ID__
       clusterDNS: [__CLUSTER_DNS__]
+{{- if eq .NodePool.ConfigItems.kuberuntu_ami_version "new" }}
+{{- if or (.NodePool.Taints) (eq .NodePool.Profile "worker-karpenter") }}
+      registerWithTaints:
+{{- range $taint := .NodePool.Taints }}
+        - key: {{$taint.Key}}
+          effect: {{$taint.Effect}}
+{{- if $taint.Value }}
+          value: {{$taint.Value }}
+{{- end }}
+{{- end }}
+{{- if eq .NodePool.Profile "worker-karpenter" }}
+        - key: karpenter.sh/unregistered
+          effect: NoExecute
+{{-  end }}
+{{-  end }}
+{{-  end }}
 
 {{- if and .Cluster.ConfigItems.vm_dirty_background_bytes .Cluster.ConfigItems.vm_dirty_bytes }}
   - owner: root:root
@@ -135,7 +153,7 @@ write_files:
       {{index $cfg 0}} = {{index $cfg 1}}
       {{- end}}
 {{- end}}
-
+{{ if eq .NodePool.ConfigItems.kuberuntu_ami_version "old" }}
   # TODO: Remove this once all nodes are running an AMI compatible with /etc/cni/net.d/10-flannel.conflist
   - owner: root:root
     path: /etc/kubernetes/cni/net.d/10-flannel.conflist
@@ -153,7 +171,7 @@ write_files:
           }
         ]
       }
-
+{{ end }}
   - owner: root:root
     path: /etc/cni/net.d/10-flannel.conflist
     content: |