templates: add toleration to control-plane components

Author: squat

hack/multi-node/bootkube-up

@@ -22,7 +22,7 @@ if [ ! -d "cluster" ]; then
   ../../_output/bin/${local_os}/bootkube render --asset-dir=cluster --api-servers=https://172.17.4.101:443 ${etcd_render_flags}
   cp user-data.sample cluster/user-data-worker
   cp user-data.sample cluster/user-data-controller
-  sed -i.bak -e '/--node-labels=node-role.kubernetes.io\/master/d' cluster/user-data-worker
+  sed -i.bak -e '/node-role.kubernetes.io\/master/d' cluster/user-data-worker
 fi
 
 # Start the VM

hack/multi-node/user-data.sample

@@ -32,6 +32,7 @@ coreos:
           --allow-privileged \
           --hostname-override=${COREOS_PUBLIC_IPV4} \
           --node-labels=node-role.kubernetes.io/master \
+          --register-with-taints=node-role.kubernetes.io/master:NoSchedule \
           --cluster_dns=10.3.0.10 \
           --cluster_domain=cluster.local
 

pkg/asset/internal/templates.go

@@ -99,6 +99,11 @@ spec:
           mountPath: /rootfs
       hostNetwork: true
       hostPID: true
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
       volumes:
       - name: dev
         hostPath:
@@ -147,12 +152,6 @@ spec:
         checkpointer.alpha.coreos.com/checkpoint: "true"
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
-      tolerations:
-      - key: "CriticalAddonsOnly"
-        operator: "Exists"
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      hostNetwork: true
       containers:
       - name: kube-apiserver
         image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
@@ -197,6 +196,16 @@ spec:
         - mountPath: /var/lock
           name: var-lock
           readOnly: false
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
       volumes:
       - name: ssl-certs-host
         hostPath:
@@ -215,7 +224,6 @@ metadata:
   name: bootstrap-kube-apiserver
   namespace: kube-system
 spec:
-  hostNetwork: true
   containers:
   - name: kube-apiserver
     image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
@@ -252,6 +260,7 @@ spec:
     - mountPath: /var/lock
       name: var-lock
       readOnly: false
+  hostNetwork: true
   volumes:
   - name: secrets
     hostPath:
@@ -281,9 +290,6 @@ spec:
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
     spec:
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      hostNetwork: true
       containers:
       - image: quay.io/coreos/kenc:48b6feceeee56c657ea9263f47b6ea091e8d3035
         name: kenc
@@ -301,6 +307,14 @@ spec:
         - /var/lock/kenc.lock
         - -c
         - "kenc -r -m iptables && kenc -m iptables"
+      hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
       volumes:
       - name: checkpoint-dir
         hostPath:
@@ -327,9 +341,6 @@ spec:
       annotations:
         checkpointer.alpha.coreos.com/checkpoint: "true"
     spec:
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
-      hostNetwork: true
       containers:
       - name: checkpoint
         image: quay.io/coreos/pod-checkpointer:8a6356146761b512cc18ab77c8a421676f641c88
@@ -357,7 +368,14 @@ spec:
         - mountPath: /var/run
           name: var-run
       hostNetwork: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
       restartPolicy: Always
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
       volumes:
       - name: etc-kubernetes
         hostPath:
@@ -384,9 +402,6 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
-      tolerations:
-      - key: "CriticalAddonsOnly"
-        operator: "Exists"
       affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -403,8 +418,6 @@ spec:
                   values:
                   - kube-contoller-manager
               topologyKey: kubernetes.io/hostname
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
       containers:
       - name: kube-controller-manager
         image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
@@ -431,6 +444,15 @@ spec:
         - name: ssl-host
           mountPath: /etc/ssl/certs
           readOnly: true
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
       volumes:
       - name: secrets
         secret:
@@ -447,7 +469,6 @@ metadata:
   name: bootstrap-kube-controller-manager
   namespace: kube-system
 spec:
-  hostNetwork: true
   containers:
   - name: kube-controller-manager
     image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
@@ -468,6 +489,7 @@ spec:
     - name: ssl-host
       mountPath: /etc/ssl/certs
       readOnly: true
+  hostNetwork: true
   volumes:
   - name: secrets
     hostPath:
@@ -506,9 +528,6 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
-      tolerations:
-      - key: "CriticalAddonsOnly"
-        operator: "Exists"
       affinity:
         podAntiAffinity:
           preferredDuringSchedulingIgnoredDuringExecution:
@@ -525,8 +544,6 @@ spec:
                   values:
                   - kube-scheduler
               topologyKey: kubernetes.io/hostname
-      nodeSelector:
-        node-role.kubernetes.io/master: ""
       containers:
       - name: kube-scheduler
         image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
@@ -540,7 +557,15 @@ spec:
             port: 10251  # Note: Using default port. Update if --port option is set differently.
           initialDelaySeconds: 15
           timeoutSeconds: 15
-
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
 `)
 
 	BootstrapSchedulerTemplate = []byte(`apiVersion: v1
@@ -549,7 +574,6 @@ metadata:
   name: bootstrap-kube-scheduler
   namespace: kube-system
 spec:
-  hostNetwork: true
   containers:
   - name: kube-scheduler
     image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
@@ -558,6 +582,7 @@ spec:
     - scheduler
     - --leader-elect=true
     - --master=http://127.0.0.1:8080
+  hostNetwork: true
 `)
 	SchedulerDisruptionTemplate = []byte(`apiVersion: policy/v1beta1
 kind: PodDisruptionBudget
@@ -588,10 +613,6 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
-      tolerations:
-      - key: "CriticalAddonsOnly"
-        operator: "Exists"
-      hostNetwork: true
       containers:
       - name: kube-proxy
         image: quay.io/coreos/hyperkube:v1.6.1_coreos.0
@@ -616,6 +637,14 @@ spec:
         - name: etc-kubernetes
           mountPath: /etc/kubernetes
           readOnly: true
+      hostNetwork: true
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
       volumes:
       - hostPath:
           path: /usr/share/ca-certificates
@@ -651,14 +680,6 @@ spec:
       annotations:
         scheduler.alpha.kubernetes.io/critical-pod: ''
     spec:
-      tolerations:
-      - key: "CriticalAddonsOnly"
-        operator: "Exists"
-      volumes:
-      - name: kube-dns-config
-        configMap:
-          name: kube-dns
-          optional: true
       containers:
       - name: kubedns
         image: gcr.io/google_containers/k8s-dns-kube-dns-amd64:1.14.1
@@ -774,6 +795,20 @@ spec:
             memory: 20Mi
             cpu: 10m
       dnsPolicy: Default  # Don't use cluster DNS.
+      nodeSelector:
+        node-role.kubernetes.io/master: ""
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
+      volumes:
+      - name: kube-dns-config
+        configMap:
+          name: kube-dns
+          optional: true
 `)
 	DNSSvcTemplate = []byte(`apiVersion: v1
 kind: Service
@@ -823,6 +858,11 @@ spec:
           valueFrom:
             fieldRef:
               fieldPath: metadata.name
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
 `)
 
 	EtcdSvcTemplate = []byte(`apiVersion: v1
@@ -849,8 +889,6 @@ metadata:
   labels:
     k8s-app: boot-etcd
 spec:
-  hostNetwork: true
-  restartPolicy: Never
   containers:
   - name: etcd
     image: quay.io/coreos/etcd:v3.1.0
@@ -870,6 +908,8 @@ spec:
         valueFrom:
           fieldRef:
             fieldPath: status.podIP
+  hostNetwork: true
+  restartPolicy: Never
 `)
 
 	KubeFlannelCfgTemplate = []byte(`apiVersion: v1
@@ -913,7 +953,6 @@ spec:
         tier: node
         app: flannel
     spec:
-      hostNetwork: true
       containers:
       - name: kube-flannel
         image: quay.io/coreos/flannel:v0.7.0-amd64
@@ -948,6 +987,12 @@ spec:
           mountPath: /etc/cni/net.d
         - name: flannel-cfg
           mountPath: /etc/kube-flannel/
+      hostNetwork: true
+      tolerations:
+      - key: node-role.kubernetes.io/master
+        operator: Equal
+        value: ""
+        effect: NoSchedule
       volumes:
         - name: run
           hostPath: