Merge pull request #382 from StackStorm/presleep

Presleep for st2auth, st2web, st2api, st2stream

Author: guzzijones

templates/deployments.yaml

@@ -60,6 +60,7 @@ spec:
           - 'sh'
           - '-ec'
           - printf "${ST2_AUTH_USERNAME}:$(openssl passwd -apr1 "${ST2_AUTH_PASSWORD}")\n" > /tmp/st2/htpasswd
+      terminationGracePeriodSeconds: {{ .Values.st2auth.terminationGracePeriodSeconds | default 30 }}
       containers:
       - name: st2auth
         image: '{{ template "stackstorm-ha.imageRepository" . }}/st2auth:{{ tpl (.Values.st2auth.image.tag | default .Values.image.tag) . }}'
@@ -96,6 +97,11 @@ spec:
           mountPath: /post-start.sh
           subPath: post-start.sh
         lifecycle:
+          preStop:
+            exec: # https://blog.laputa.io/graceful-shutdown-in-kubernetes-85f1c8d586da
+              # kubeproxy and kubelet both race  
+              # wait a bit to ensure kubeproxy has removed the iptables rule.
+              command: [ "sleep", {{ .Values.st2auth.preStopSleep | toString }} ]
           postStart:
             exec:
               command: ["/bin/bash", "/post-start.sh"]
@@ -179,6 +185,7 @@ spec:
       {{- if and .Values.st2.packs.images (not .Values.st2.packs.volumes.enabled) }}
         {{- include "stackstorm-ha.packs-initContainers" . | nindent 6 }}
       {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.st2api.terminationGracePeriodSeconds | default 30 }}
       containers:
       - name: st2api
         image: '{{ template "stackstorm-ha.imageRepository" . }}/st2api:{{ tpl (.Values.st2api.image.tag | default .Values.image.tag) . }}'
@@ -220,6 +227,11 @@ spec:
           mountPath: /post-start.sh
           subPath: post-start.sh
         lifecycle:
+          preStop:
+            exec: # https://blog.laputa.io/graceful-shutdown-in-kubernetes-85f1c8d586da
+              # kubeproxy and kubelet both race  
+              # wait a bit to ensure kubeproxy has removed the iptables rule.
+              command: [ "sleep", {{ .Values.st2api.preStopSleep | toString }} ]
           postStart:
             exec:
               command: ["/bin/bash", "/post-start.sh"]
@@ -305,6 +317,7 @@ spec:
       initContainers:
       {{- include "stackstorm-ha.init-containers-wait-for-db" . | nindent 6 }}
       {{- include "stackstorm-ha.init-containers-wait-for-mq" . | nindent 6 }}
+      terminationGracePeriodSeconds: {{ .Values.st2stream.terminationGracePeriodSeconds | default 30 }}
       containers:
       - name: st2stream
         image: '{{ template "stackstorm-ha.imageRepository" . }}/st2stream:{{ tpl (.Values.st2stream.image.tag | default .Values.image.tag) . }}'
@@ -337,6 +350,11 @@ spec:
           mountPath: /post-start.sh
           subPath: post-start.sh
         lifecycle:
+          preStop:
+            exec: # https://blog.laputa.io/graceful-shutdown-in-kubernetes-85f1c8d586da
+              # kubeproxy and kubelet both race  
+              # wait a bit to ensure kubeproxy has removed the iptables rule.
+              command: [ "sleep", {{ .Values.st2stream.preStopSleep | toString }} ]
           postStart:
             exec:
               command: ["/bin/bash", "/post-start.sh"]
@@ -405,6 +423,7 @@ spec:
       {{- if .Values.image.pullSecret }}
       - name: {{ .Values.image.pullSecret }}
       {{- end }}
+      terminationGracePeriodSeconds: {{ .Values.st2web.terminationGracePeriodSeconds | default 30 }}
       containers:
       - name: st2web
         image: '{{ template "stackstorm-ha.imageRepository" . }}/st2web:{{ tpl (.Values.st2web.image.tag | default .Values.image.tag) . }}'
@@ -457,6 +476,11 @@ spec:
             mountPath: /post-start.sh
             subPath: post-start.sh
         lifecycle:
+          preStop:
+            exec: # https://blog.laputa.io/graceful-shutdown-in-kubernetes-85f1c8d586da
+              # kubeproxy and kubelet both race  
+              # wait a bit to ensure kubeproxy has removed the iptables rule.
+              command: [ "sleep", {{ .Values.st2web.preStopSleep | toString }} ]
           postStart:
             exec:
               command: ["/bin/bash", "/post-start.sh"]

values.yaml

@@ -381,6 +381,7 @@ st2web:
   # The pod will not be marked as "running" until this script completes successfully.
   # see: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
   postStartScript: ""
+  preStopSleep: "10"
 
 # https://docs.stackstorm.com/reference/ha.html#st2auth
 # Multiple st2auth processes can be behind a load balancer in an active-active configuration.
@@ -416,6 +417,7 @@ st2auth:
   # The pod will not be marked as "running" until this script completes successfully.
   # see: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
   postStartScript: ""
+  preStopSleep: "10"
   # mount extra volumes on the st2auth pod(s) (primarily useful for custom logging conf)
   ## Note that Helm templating is supported in 'mount' and 'volume'
   extra_volumes: []
@@ -455,6 +457,7 @@ st2api:
   # The pod will not be marked as "running" until this script completes successfully.
   # see: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
   postStartScript: ""
+  preStopSleep: "10"
   # mount extra volumes on the st2api pod(s) (primarily useful for custom logging conf)
   ## Note that Helm templating is supported in 'mount' and 'volume'
   extra_volumes: []
@@ -494,6 +497,7 @@ st2stream:
   # The pod will not be marked as "running" until this script completes successfully.
   # see: https://kubernetes.io/docs/concepts/containers/container-lifecycle-hooks/
   postStartScript: ""
+  preStopSleep: "10"
   # mount extra volumes on the st2stream pod(s) (primarily useful for custom logging conf)
   ## Note that Helm templating is supported in 'mount' and 'volume'
   extra_volumes: []