fix(install): Increase stop timeout to 60 seconds (#731)

This is to ensure clean shutdown of Celery, with fully drained queues. This is needed as versions may change the event format and not be backwards compatible. FWIW this is a hacky workaround without a strong guarantee that the queues will be empty. Ideally we'd shutdown everything first, spin up the workers and check for queues being drained every second or so.

Author: BYK

install.sh

@@ -15,6 +15,11 @@ MIN_DOCKER_VERSION='19.03.6'
 MIN_COMPOSE_VERSION='1.24.1'
 MIN_RAM=2400 # MB
 
+# Increase the default 10 second SIGTERM timeout
+# to ensure celery queues are properly drained 
+# between upgrades as task signatures may change across
+# versions
+STOP_TIMEOUT=60 # seconds
 SENTRY_CONFIG_PY='sentry/sentry.conf.py'
 SENTRY_CONFIG_YML='sentry/config.yml'
 SYMBOLICATOR_CONFIG_YML='symbolicator/config.yml'
@@ -76,7 +81,7 @@ cleanup () {
   fi
 
   if [[ -z "$MINIMIZE_DOWNTIME" ]]; then
-    $dc stop &> /dev/null
+    $dc stop -t $STOP_TIMEOUT &> /dev/null
   fi
 }
 trap_with_arg cleanup ERR INT TERM EXIT
@@ -224,9 +229,9 @@ if [[ -n "$MINIMIZE_DOWNTIME" ]]; then
 else
   # Clean up old stuff and ensure nothing is working while we install/update
   # This is for older versions of on-premise:
-  $dc -p onpremise down --rmi local --remove-orphans
+  $dc -p onpremise down -t $STOP_TIMEOUT --rmi local --remove-orphans
   # This is for newer versions
-  $dc down --rmi local --remove-orphans
+  $dc down -t $STOP_TIMEOUT --rmi local --remove-orphans
 fi
 
 ZOOKEEPER_SNAPSHOT_FOLDER_EXISTS=$($dcr zookeeper bash -c 'ls 2>/dev/null -Ubad1 -- /var/lib/zookeeper/data/version-2 | wc -l | tr -d '[:space:]'')