Fix race conditions while using many nodes with docker-compose

Author: elgalu

CHANGELOG.md

@@ -7,6 +7,30 @@ Note image ids also change after scm-source.json has being updated which trigger
 ###### To get container versions
     docker exec grid versions
 
+## TBD_DOCKER_TAG
+ + Date: TBD_DATE
+ + Fix race conditions while using many nodes with docker-compose
+ + Image tag details:
+  + Selenium: vTBD_SELENIUM_VERSION (TBD_SELENIUM_REVISION)
+  + Chrome stable:  TBD_CHROME_STABLE
+  + Firefox stable: TBD_FIREFOX_STABLE
+  + Chromedriver: TBD_CHROME_DRIVER (TBD_CHROMEDRIVER_COMMIT)
+  + Java: TBD_JAVA_VENDOR Java TBD_JAVA_BUILD
+  + Timezone: TBD_TIME_ZONE
+  + FROM ubuntu:UBUNTU_FLAVOR-UBUNTU_DATE
+  + Python: TBD_PYTHON_VERSION
+  + Sauce Connect TBD_SAUCE_CONNECT_VERS, build TBD_SAUCE_CONNECT_BUILD TBD_SAUCE_CONNECT_REVISION
+  + BrowserStack Local version TBD_BROWSER_STACK_VERSION
+  + Tested on kernel dev host: 4.4.0-29-generic x86_64
+  + Tested on kernel CI  host: TBD_HOST_UNAME
+  + Built at dev host with: Docker version 1.11.2, build b9f10c9
+  + Built at CI  host with: Docker version TBD_DOCKER_VERS, build TBD_DOCKER_BUILD
+  + Built at dev host with: Docker Compose version 1.7.1, build 0a9ab35
+  + Built at CI  host with: Docker Compose version TBD_DOCKER_COMPOSE_VERS, build TBD_DOCKER_COMPOSE_BUILD
+  + Image size: TBD_IMAGE_SIZE
+  + Digest: TBD_DIGEST
+  + Image ID: TBD_IMAGE_ID
+
 ## 2.53.1a
  + Date: 2016-06-30
  + Upgrade Selenium to 2.53.1

CONTRIBUTING.md

@@ -3,24 +3,24 @@
 ## Local
 For pull requests or local commits:
 
-    docker rm -vf grid; ./test/before_install_build && ./test/install && ./test/script && docker tag selenium:latest elgalu/selenium:latest
+    ./test/before_install_build && ./test/install && ./test/script && docker tag selenium:latest elgalu/selenium:latest
     docker exec grid versions && ./test/after_script
     open ./images/grid_console.png #to verify the versions are correct
     git checkout ./images/grid_console.png && open ./videos/chrome/test.mkv
     travis lint #if you changed .travis.yml
-    git checkout -b tmp-2.53.1a #name your branch according to your changes
+    git checkout -b tmp-2.53.1b #name your branch according to your changes
     #git add ... git commit ... git push ... open pull request
 
 For repository owners only:
 
-    git commit -m "Selenium 2.53.1 & Firefox 47.0.1 #102 & docker-compose #108 & Upgrade ubuntu 20160629"
+    git commit -m "Fix race conditions while using many nodes with docker-compose"
     git tag -d latest #tag latest will be updated from TravisCI
-    git tag 2.53.1a && git push origin tmp-2.53.1a && git push --tags
+    git tag 2.53.1b && git push origin tmp-2.53.1b && git push --tags
 
 -- Wait for Travis to pass OK
 -- Make sure changes got merged into master by elgalubot
 
-    git checkout master && git pull && git branch -d tmp-2.53.1a && git push origin --delete tmp-2.53.1a
+    git checkout master && git pull && git branch -d tmp-2.53.1b && git push origin --delete tmp-2.53.1b
 
 -- Re-add TBD_* section in CHANGELOG.md starting with TBD_DOCKER_TAG
 -- Upgrade release tag in github.com with latest CHANGELOG.md
@@ -37,9 +37,9 @@ Keep certain bins if chrome version changed for example:
 ## Retry
 Failed in Travis? retry
 
-    git tag -d 2.53.1a && git push origin :2.53.1a
+    git tag -d 2.53.1b && git push origin :2.53.1b
     #git add ...
-    git commit --amend && git tag 2.53.1a && git push --force origin tmp-2.53.1a && git push --tags
+    git commit --amend && git tag 2.53.1b && git push --force origin tmp-2.53.1b && git push --tags
 
 ## Docker push from Travis CI
 Travis [steps](https://docs.travis-ci.com/user/docker/#Pushing-a-Docker-Image-to-a-Registry) involve `docker login` and docker credentials encryptions.

Dockerfile

@@ -911,8 +911,8 @@ ENV DEFAULT_SELENIUM_HUB_PORT="24444" \
     DEFAULT_NOVNC_PORT="26080" \
     DEFAULT_SSHD_PORT="22222" \
     DEFAULT_SAUCE_LOCAL_SEL_PORT="4445" \
-    DEFAULT_SUPERVISOR_HTTP_PORT="29001" \
-    DEFAULT_DISP_N="10"
+    DEFAULT_SUPERVISOR_HTTP_PORT="29001"
+    # DEFAULT_DISP_N="10"
 
 # Commented for now; all these versions are still available at
 #   https://github.com/elgalu/docker-selenium/releases/tag/2.47.1m
@@ -935,6 +935,10 @@ ENV FIREFOX_VERSION="${FF_VER}" \
   MEM_JAVA_PERCENT=80 \
   # Max amount of time to wait on Xvfb or Xmanager while retrying
   WAIT_FOREGROUND_RETRY="1s" \
+  # Supervisor processes retry attemps (0 means do not retry)
+  XVFB_STARTRETRIES=0 \
+  XMANAGER_STARTRETRIES=0 \
+  XMANAGER_STARTSECS=0 \
   # Max amount of time to wait for other processes dependencies
   WAIT_TIMEOUT="25s" \
   SCREEN_WIDTH=1900 \
@@ -1043,6 +1047,7 @@ ENV FIREFOX_VERSION="${FF_VER}" \
   VIDEO_CHUNKS_MAX=999 \
   VIDEOS_DIR="${NORMAL_USER_HOME}/videos" \
   # You can choose what X manager to use
+  #  fluxbox | openbox
   XMANAGER="fluxbox" \
   # Sauce Labs tunneling. Naming is required: SAUCE_TUNNEL_ID
   SAUCE_TUNNEL="false" \

bin/entry.sh

@@ -172,7 +172,7 @@ fi
 
 #----------------------------------------
 # Remove lock files, thanks @garagepoort
-clear_x_locks.sh
+# clear_x_locks.sh
 
 #--------------------------------
 # Improve etc/hosts and fix dirs
@@ -195,47 +195,57 @@ function get_free_display() {
 
   # Get a list of socket DISPLAYs already used
   netstat -nlp | grep -Po '(?<=\/tmp\/\.X11-unix\/X)([0-9]+)' | sort -u > /tmp/netstatX11.log
-  #DEBUG: cat /tmp/netstatX11.log 1>&3
-
-  # -s  file is not zero size
-  if [ -s /tmp/netstatX11.log ]; then
-    # important: while loops are executed in a subshell
-    #  var assignments will be lost unless using <<<
-    # using 11.0 12.3 1.8 and so on didn't work, left as a reference
-    #  local pythonCmd="from random import shuffle;list1 = list(range($MAX_DISPLAY_SEARCH));shuffle(list1);list2 = [x/10 for x in list1];str_res = ' '.join(str(e) for e in list2);print (str_res)"
-    local pythonCmd="from random import shuffle;list1 = list(range($MAX_DISPLAY_SEARCH));shuffle(list1);print (' '.join(str(e) for e in list1))"
-    local displayNums=$(python -c "${pythonCmd}")
-    # Always find a free DISPLAY port starting with current DISP_N if it was provided
-    [ "${DISP_N}" != "-1" ] && displayNums="${DISP_N} ${displayNums}"
-    IFS=' ' read -r -a arrayDispNums <<< "$displayNums"
-    for find_display_num in ${arrayDispNums[@]}; do
-      # read -r Do not treat a backslash character in any special way.
-      #         Consider each backslash to be part of the input line.
+  [ ! -s /tmp/netstatX11.log ] && echo "-- INFO: Emtpy file /tmp/netstatX11.log" 1>&3
+
+  # important: while loops are executed in a subshell
+  #  var assignments will be lost unless using <<<
+  # using 11.0 12.3 1.8 and so on didn't work, left as a reference
+  #  local pythonCmd="from random import shuffle;list1 = list(range($MAX_DISPLAY_SEARCH));shuffle(list1);list2 = [x/10 for x in list1];str_res = ' '.join(str(e) for e in list2);print (str_res)"
+  local pythonCmd="from random import shuffle;list1 = list(range($MAX_DISPLAY_SEARCH));shuffle(list1);print (' '.join(str(e) for e in list1))"
+  local displayNums=$(python -c "${pythonCmd}")
+  # Always find a free DISPLAY port starting with current DISP_N if it was provided
+  [ "${DISP_N}" != "-1" ] && displayNums="${DISP_N} ${displayNums}"
+  IFS=' ' read -r -a arrayDispNums <<< "$displayNums"
+  for find_display_num in ${arrayDispNums[@]}; do
+    # read -r Do not treat a backslash character in any special way.
+    #         Consider each backslash to be part of the input line.
+    # -s  file is not zero size
+    if [ -s /tmp/netstatX11.log ]; then
       while read read_disp_num ; do
         if [ "${read_disp_num}" = "${find_display_num}" ]; then
-          echo "-- WARN: DISPLAY=${find_display_num} is taken, searching for another..." 1>&3
+          echo "-- WARN: DISPLAY=:${find_display_num} is taken, searching for another..." 1>&3
           selected_disp_num="-1"
           break
         elif [ "${selected_disp_num}" = "-1" ]; then
           selected_disp_num="${find_display_num}"
-          echo "-- INFO: Possible free DISPLAY=${find_display_num}" 1>&3
+          echo "-- INFO: Possible free DISPLAY=:${find_display_num}" 1>&3
           # echo "-- DEBUG: Updated selected_disp_num=$selected_disp_num" 1>&3
           # echo "-- DEBUG: WAS read_disp_num=$read_disp_num" 1>&3
         fi
       done <<< "$(cat /tmp/netstatX11.log)"
-      # echo "-- DEBUG: selected_disp_num=$selected_disp_num" 1>&3
-      [ "${selected_disp_num}" != "-1" ] && break
-      # echo "-- DEBUG: find_display_num=$find_display_num" 1>&3
-      if [ ${find_display_num} -gt ${MAX_DISPLAY_SEARCH} ]; then
-        echo "-- ERROR: Entered in an infinite loop at $0 after netstat" 1>&2 1>&3
-        break
+    else
+      # echo "-- INFO: Emtpy file /tmp/netstatX11.log" 1>&3
+      # selected_disp_num="${DEFAULT_DISP_N}"
+      selected_disp_num="${find_display_num}"
+    fi
+    if [ "${selected_disp_num}" != "-1" ]; then
+      # If we can already use that display it means there is already some
+      # Xvfb there which means we need to keep looking for a free one.
+      export DISPLAY=":${find_display_num}"
+      if xsetroot -cursor_name left_ptr -fg white -bg black > /dev/null 2>&1; then
+        echo "-- WARN: DISPLAY=:${find_display_num} is already being used, skip it..." 1>&3
+        selected_disp_num="-1"
       fi
-    done
-  else
-    echo "-- INFO: Emtpy file /tmp/netstatX11.log" 1>&3
-    selected_disp_num="${DEFAULT_DISP_N}"
-  fi
-  [ "${selected_disp_num}" = "-1" ] || echo "-- INFO: Found free DISPLAY=${selected_disp_num}" 1>&3
+    fi
+    if [ "${selected_disp_num}" != "-1" ]; then
+      break
+    elif [ ${find_display_num} -gt ${MAX_DISPLAY_SEARCH} ]; then
+      echo "-- ERROR: Entered in an infinite loop at $0 after netstat" 1>&2 1>&3
+      selected_disp_num="-1"
+      break
+    fi
+  done
+  [ "${selected_disp_num}" = "-1" ] || echo "-- INFO: Found free DISPLAY=:${selected_disp_num}" 1>&3
 
   echo ${selected_disp_num}
 }

test/compose-test.sh

@@ -23,14 +23,20 @@ die () {
 [ -z "${SELENIUM_HUB_PORT}" ] && die "Required env var SELENIUM_HUB_PORT"
 [ -z "${WAIT_ALL_DONE}" ] && export WAIT_ALL_DONE="40s"
 
+SLEEP_LOCALLY=0
+SLEEP_TRAVIS=3
+
+# Ensure clean
+docker-compose -p selenium down || true
+
 # Compose up!
 docker-compose -p selenium scale hub=1 chrome=${NUM_NODES} firefox=${NUM_NODES}
 
 # FIXME: We still need to wait a bit because the nodes registration is not
 #        being waited on wait_all_done script :(
 #        mabe related to issue #83
-sleep 3
-[ "${TRAVIS}" = "true" ] && sleep 3
+sleep ${SLEEP_LOCALLY}
+[ "${TRAVIS}" = "true" ] && sleep ${SLEEP_TRAVIS}
 
 # Wait then show errors, if any
 if ! docker exec selenium_hub_1 wait_all_done ${WAIT_ALL_DONE}; then
@@ -55,8 +61,8 @@ done
 # FIXME: We still need to wait a bit because the nodes registration is not
 #        being waited on wait_all_done script :(
 #        mabe related to issue #83
-sleep 4
-[ "${TRAVIS}" = "true" ] && sleep 4
+sleep ${SLEEP_LOCALLY}
+[ "${TRAVIS}" = "true" ] && sleep ${SLEEP_TRAVIS}
 
 # Tests can run anywere, in the hub, in the host, doesn't matter
 for i in $(seq 1 ${PARAL_TESTS}); do
@@ -68,8 +74,8 @@ for i in $(seq 1 ${PARAL_TESTS}); do
 done
 
 # sleep a moment to let the UI tests start
-sleep 4
-[ "${TRAVIS}" = "true" ] && sleep 4
+sleep ${SLEEP_LOCALLY}
+[ "${TRAVIS}" = "true" ] && sleep ${SLEEP_TRAVIS}
 
 # not so verbose from here
 set +x
@@ -88,7 +94,7 @@ for i in $(seq 1 ${NUM_NODES}); do
 done
 
 # Cleanup
-docker-compose down
+docker-compose -p selenium down
 
 # Results
 if [ "$FAIL_COUNT" == "0" ]; then

test/install_grid

@@ -3,6 +3,9 @@
 # set -e: exit asap if a command exits with a non-zero status
 set -e
 
+# Ensure clean env
+docker rm -vf grid || true
+
 docker run --name=grid -d -e VIDEO=true -v /dev/shm:/dev/shm selenium
 docker exec grid wait_all_done 40s
 docker exec grid versions

xmanager/bin/start-xmanager.sh

@@ -23,8 +23,17 @@ die () {
 # Wait for this process dependencies
 timeout --foreground ${WAIT_TIMEOUT} wait-xvfb.sh
 
+function shutdown {
+  echo "Trapped SIGTERM/SIGINT so shutting down $0 gracefully..."
+  exit 0
+}
+
+# Run function shutdown() when this process a killer signal
+trap shutdown SIGTERM SIGINT SIGKILL
+
 function start_fluxbox() {
-  fluxbox -display ${DISPLAY} -verbose \
+  # http://stackoverflow.com/a/21028200/511069
+  fluxbox -display "${DISPLAY}.0" -verbose \
     1> "${LOGS_DIR}/fluxbox-tryouts-stdout.log" \
     2> "${LOGS_DIR}/fluxbox-tryouts-stderr.log" &
 }
@@ -37,25 +46,26 @@ elif [ "${XMANAGER}" = "fluxbox" ]; then
   i=0
   stat_failed=true
   while true ; do
-    let i=${i}+1
-    if ! start_fluxbox; then
-      echo "-- WARN: start_fluxbox() failed!" 1>&3
-    fi
-    timeout --foreground ${WAIT_TIMEOUT} wait-xvfb.sh
-    if timeout --foreground "${WAIT_FOREGROUND_RETRY}" wait-xmanager.sh &> "${LOGS_DIR}/wait-xmanager-stdout.log"; then
-      stat_failed=false
-      break
-    else
-      echo "-- WARN: wait-xmanager.sh failed! for DISPLAY=${DISPLAY}" 1>&3
-      killall fluxbox || true
-    fi
-    if [ ${i} -gt 10 ]; then
-      echoerr "-- ERROR: Failed to start Fluxbox at $0 after many retries."
-      break
-    fi
+    while true ; do
+      let i=${i}+1
+      if ! start_fluxbox; then
+        echo "-- WARN: start_fluxbox() failed!" 1>&3
+      fi
+      if timeout --foreground "${WAIT_FOREGROUND_RETRY}" wait-xmanager.sh &> "${LOGS_DIR}/wait-xmanager-stdout.log"; then
+        stat_failed=false
+        break
+      else
+        echo "-- WARN: wait-xmanager.sh failed! for DISPLAY=${DISPLAY}" 1>&3
+        killall fluxbox || true
+      fi
+      if [ ${i} -gt 3 ]; then
+        echoerr "-- ERROR: Failed to start Fluxbox at $0 after many retries."
+        break
+      fi
+    done
+    [ "${stat_failed}" != "true" ] && wait
+    stat_failed=true
   done
-  [ "${stat_failed}" = "true" ] && die "Failed to start_fluxbox()."
-  wait
 else
   die "The chosen X manager is not supported: '${XMANAGER}'"
 fi

xmanager/etc/supervisor/conf.d/xmanager.conf

@@ -29,12 +29,12 @@ autorestart=false
 ;particular amount of time.
 ;So using custom wait-xxxx.sh scripts to perform a more efficient
 ;active waiting until https://github.com/Supervisor/supervisor/issues/584
-startsecs=0
+startsecs=%(ENV_XMANAGER_STARTSECS)s
 
 ;The number of serial failure attempts that supervisord will allow when
 ;attempting to start the program before giving up and puting the process
 ;into an FATAL state.
-startretries=0
+startretries=%(ENV_XMANAGER_STARTRETRIES)s
 
 ;Logs
 redirect_stderr=false

xterm/bin/start-xterm.sh

@@ -1,8 +1,12 @@
 #!/usr/bin/env bash
 
+# Open new file descriptors that redirects to stderr/stdout
+exec 3>&1
+exec 4>&2
+
 # echo fn that outputs to stderr http://stackoverflow.com/a/2990533/511069
 echoerr() {
-  cat <<< "$@" 1>&2;
+  cat <<< "$@" 1>&4;
 }
 
 # print error and exit
@@ -26,6 +30,9 @@ shutdown () {
   die "Some processes failed to start so quitting."
 }
 
+# clean status file
+echo "" > ${DOCKER_SELENIUM_STATUS}
+
 # timeout runs the given command and kills it if it is still running
 # after the specified time interval:
 #  http://www.gnu.org/software/coreutils/manual/coreutils.html#timeout-invocation
@@ -79,5 +86,6 @@ x-terminal-emulator -ls  \
 # Join them in 1 bash line to avoid supervisor split them in debug output
 # this output is used to signal docker-selenium is ready for testing
 echo -e "\nContainer docker internal IP: $CONTAINER_IP\n"
+echo -e "\nContainer docker internal IP: $CONTAINER_IP\n" 1>&3
 
 wait

xvfb/etc/supervisor/conf.d/xvfb.conf

@@ -37,7 +37,7 @@ startsecs=0
 ;The number of serial failure attempts that supervisord will allow when
 ;attempting to start the program before giving up and puting the process
 ;into an FATAL state.
-startretries=0
+startretries=%(ENV_XVFB_STARTRETRIES)s
 
 ;Logs
 redirect_stderr=false