Add node failure tolerations to all service operators

This change adds 120s tolerations for node.kubernetes.io/not-ready
and unreachable taints to reduce pod failover during a node failure.

The total eviction time is  ~160s (5min+ default). 120s was choosen
to prevents pod rescheduling e.g. on kubelet restarts or network issues

Jira: OSPRH-18450

Signed-off-by: Martin Schuppert <[email protected]>

Author: stuggi

bindata/operator/managers.yaml

@@ -85,6 +85,15 @@ spec:
         runAsNonRoot: true
       serviceAccountName: {{ .Name }}-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
 {{- if isEnvVarTrue .Deployment.Manager.Env "ENABLE_WEBHOOKS" }}
       volumes:
       - name: cert

bindata/operator/operator.yaml

@@ -133,6 +133,15 @@ spec:
         runAsNonRoot: true
       serviceAccountName: openstack-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - effect: NoExecute
+        key: node.kubernetes.io/not-ready
+        operator: Exists
+        tolerationSeconds: 120
+      - effect: NoExecute
+        key: node.kubernetes.io/unreachable
+        operator: Exists
+        tolerationSeconds: 120
       volumes:
       - name: cert
         secret:

bindata/operator/rabbit.yaml

@@ -46,3 +46,12 @@ spec:
             memory: {{ .RabbitmqOperator.Deployment.Manager.Resources.Requests.Memory }}
       serviceAccountName: rabbitmq-cluster-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120

config/manager/manager.yaml

@@ -72,3 +72,12 @@ spec:
             customRequests: replace_me #NOTE: this is used via the Makefile to inject a custom template that kustomize won't allow
       serviceAccountName: openstack-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120

config/operator/deployment/deployment.yaml

@@ -106,3 +106,12 @@ spec:
             memory: 128Mi
       serviceAccountName: openstack-operator-controller-operator
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120

config/operator/managers.yaml

@@ -85,6 +85,15 @@ spec:
         runAsNonRoot: true
       serviceAccountName: {{ .Name }}-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
 {{- if isEnvVarTrue .Deployment.Manager.Env "ENABLE_WEBHOOKS" }}
       volumes:
       - name: cert

config/operator/rabbit.yaml

@@ -46,3 +46,12 @@ spec:
             memory: {{ .RabbitmqOperator.Deployment.Manager.Resources.Requests.Memory }}
       serviceAccountName: rabbitmq-cluster-operator-controller-manager
       terminationGracePeriodSeconds: 10
+      tolerations:
+      - key: "node.kubernetes.io/not-ready"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120
+      - key: "node.kubernetes.io/unreachable"
+        operator: "Exists"
+        effect: "NoExecute"
+        tolerationSeconds: 120