Automatic health check and restart if some spawned iip processes are exited

Author: urubens

configure.ac

@@ -1,6 +1,6 @@
 # Process this file with autoconf to produce a configure script.
 
-AC_INIT(iipsrv,1.3.2)
+AC_INIT(iipsrv,1.4.0)
 AC_CONFIG_MACRO_DIR([m4])
 AM_INIT_AUTOMAKE
 

docker/Dockerfile

@@ -20,6 +20,7 @@ FROM ${FROM_NAMESPACE}/base:$FROM_VERSION
 
 RUN apt-get -y update && \
     apt-get install -y \
+      cron \
       git \
       libjpeg8-dev \
       libmemcached-dev \
@@ -30,6 +31,7 @@ RUN apt-get -y update && \
       psmisc \
       autoconf \
       libtool \
+      rsyslog \
       spawn-fcgi \
       software-properties-common && \
       apt-get -y build-dep openslide && \
@@ -71,6 +73,12 @@ RUN cd /opt/iipsrv && \
 COPY deploy.sh /tmp/deploy.sh
 RUN chmod +x /tmp/deploy.sh
 
+COPY start-iip.sh /tmp/start-iip.sh
+RUN chmod +x /tmp/start-iip.sh
+
+COPY check-status.sh /tmp/check-status.sh
+RUN chmod +x /tmp/check-status.sh
+
 RUN ldconfig -v
 
 ENTRYPOINT ["/tmp/deploy.sh"]

docker/check-status.sh

@@ -0,0 +1,8 @@
+. /tmp/iip-configuration.sh
+
+COUNT=$(ps aux | grep /opt/iipsrv/src/iipsrv.fcgi | grep -v grep | wc -l)
+if [ $COUNT -lt $NB_IIP_PROCESS ]; then
+    echo "$COUNT / $NB_IIP_PROCESS on $(date)"
+    killall -9 /opt/iipsrv/src/iipsrv.fcgi
+    /tmp/start-iip.sh
+fi;

docker/deploy.sh

@@ -26,13 +26,9 @@ echo "  create 640 root root"           >> /etc/logrotate.d/iip
 echo "  su root root"                   >> /etc/logrotate.d/iip
 echo "}"                                >> /etc/logrotate.d/iip
 
-# IIP configuration options
-export VERBOSITY=10
-export LOGFILE=/tmp/iip.out
 . /tmp/iip-configuration.sh
 sysctl -w net.core.somaxconn=2048
 
-env
 
 # Configure Nginx
 PORT=9000
@@ -46,12 +42,15 @@ mv /tmp/nginx.conf.sample /usr/local/nginx/conf/nginx.conf
 echo "start nginx"
 /usr/local/nginx/sbin/nginx &
 
-# Spawn FCGI processes
-COUNTER=0
-while [  $COUNTER -lt $NB_IIP_PROCESS ]; do
-    echo "spawn process"
-    spawn-fcgi -f /opt/iipsrv/src/iipsrv.fcgi -a 127.0.0.1 -p $(($PORT+$COUNTER))
-    let COUNTER=COUNTER+1
-done
+/tmp/start-iip.sh
+
+touch /tmp/crontab
+echo "NB_IIP_PROCESS=$NB_IIP_PROCESS" >> /tmp/crontab
+echo "*/1 * * * * /bin/bash /tmp/check-status.sh >> /tmp/cron.out" >> /tmp/crontab
+crontab /tmp/crontab
+rm /tmp/crontab
+
+service rsyslog restart
+service cron restart
 
 tail -F /tmp/iip.out

docker/start-iip.sh

@@ -0,0 +1,11 @@
+export VERBOSITY=10
+export LOGFILE=/tmp/iip.out
+. /tmp/iip-configuration.sh
+
+PORT=9000
+COUNTER=0
+while [  $COUNTER -lt $NB_IIP_PROCESS ]; do
+    echo "spawn process"
+    spawn-fcgi -f /opt/iipsrv/src/iipsrv.fcgi -a 127.0.0.1 -p $(($PORT+$COUNTER))
+    let COUNTER=COUNTER+1
+done