geckodriver, Ubuntu 20160923, chromedriver.log, DBUS magic

Author: elgalu

.travis.yml

@@ -25,11 +25,11 @@ matrix:
     #   com.docker.docker Incompatible CPU detected
     #   Docker requires a processor with virtualization capabilities
     - env:
-        - DOCKER_VERSION="1.12.1-rc2"
-          DOCKER_COMPOSE_VERSION="1.8.0"
+        - DOCKER_VERSION="1.12.1"
+          DOCKER_COMPOSE_VERSION="1.8.1"
           OPERATING_SYSTEM="osx"
   include:
-    # Linux with docker:stable compose:stable/older
+    # Linux with docker:stable compose:stable
     # Use DOCKER_PUSH_LATEST=false to debug
     - os: linux
       sudo: required
@@ -38,19 +38,19 @@ matrix:
         - docker
       env:
         - DOCKER_VERSION="stable"
-          DOCKER_COMPOSE_VERSION="1.8.0"
+          DOCKER_COMPOSE_VERSION="1.8.1"
           DOCKER_PUSH=true
           DOCKER_PUSH_LATEST=true
-    # Linux with docker:beta compose:stable/older
+    # Linux with docker:beta/older compose:beta/older
     - os: linux
       sudo: required
       dist: trusty
       services:
         - docker
       env:
-        - DOCKER_VERSION="1.12.1-rc2"
+        - DOCKER_VERSION="1.12.0"
           DOCKER_COMPOSE_VERSION="1.8.0"
-    # Linux with docker:stable compose:beta
+    # Linux with docker:stable compose:beta/older
     - os: linux
       sudo: required
       dist: trusty
@@ -59,21 +59,21 @@ matrix:
       env:
         - DOCKER_VERSION="stable"
           DOCKER_COMPOSE_VERSION="1.8.0"
-    # Linux with docker:beta compose:beta
+    # Linux with docker:beta/older compose:stable
     - os: linux
       sudo: required
       dist: trusty
       services:
         - docker
       env:
-        - DOCKER_VERSION="1.12.1-rc2"
-          DOCKER_COMPOSE_VERSION="1.8.0"
+        - DOCKER_VERSION="1.12.0"
+          DOCKER_COMPOSE_VERSION="1.8.1"
     # Mac OSX: only test native docker version
     - os: osx
       osx_image: xcode8
       env:
-        - DOCKER_VERSION="1.12.1-rc2"
-          DOCKER_COMPOSE_VERSION="1.8.0"
+        - DOCKER_VERSION="1.12.1"
+          DOCKER_COMPOSE_VERSION="1.8.1"
           OPERATING_SYSTEM="osx"
 
 before_install:

CHANGELOG.md

@@ -7,6 +7,34 @@ Note image ids also change after scm-source.json has being updated which trigger
 ###### To get container versions
     docker exec grid versions
 
+## TBD_DOCKER_TAG
+ + Date: TBD_DATE
+ + Set chromedriver --log-path='/var/log/cont/chromedriver.log'
+ + Add geckodriver for Firefox >= 48 (future releases)
+ + Upgrade Ubuntu xenial to 20160923
+ + Perhaps fixed DBUS mistery with at entry.sh
+ + Client side: docker-compose from 1.8.0 to 1.8.1
+ + Image tag details:
+  + Selenium: vTBD_SELENIUM_VERSION (TBD_SELENIUM_REVISION)
+  + Chrome stable:  TBD_CHROME_STABLE
+  + Firefox stable: TBD_FIREFOX_STABLE
+  + Chromedriver: TBD_CHROME_DRIVER (TBD_CHROMEDRIVER_COMMIT)
+  + Java: TBD_JAVA_VENDOR Java TBD_JAVA_BUILD
+  + Timezone: TBD_TIME_ZONE
+  + FROM ubuntu:UBUNTU_FLAVOR-UBUNTU_DATE
+  + Python: TBD_PYTHON_VERSION
+  + Sauce Connect TBD_SAUCE_CONNECT_VERS, build TBD_SAUCE_CONNECT_BUILD TBD_SAUCE_CONNECT_REVISION
+  + BrowserStack Local version TBD_BROWSER_STACK_VERSION
+  + Tested on kernel dev host: 4.4.0-38-generic x86_64
+  + Tested on kernel CI  host: TBD_HOST_UNAME
+  + Built at dev host with: Docker version 1.12.1, build 23cf638
+  + Built at CI  host with: Docker version TBD_DOCKER_VERS, build TBD_DOCKER_BUILD
+  + Built at dev host with: Docker Compose version 1.8.1, build 878cff1
+  + Built at CI  host with: Docker Compose version TBD_DOCKER_COMPOSE_VERS, build TBD_DOCKER_COMPOSE_BUILD
+  + Image size: TBD_IMAGE_SIZE
+  + Digest: TBD_DIGEST
+  + Image ID: TBD_IMAGE_ID
+
 ## 2.53.1x
  + Date: 2016-09-15
  + Upgrade Chrome stable patch 53.0.2785.116

CONTRIBUTING.md

@@ -6,18 +6,18 @@ For pull requests or local commits:
     time (./test/bef && ./test/install && ./test/script_start && ./test/script_end) ; beep
     docker exec grid versions && ./test/after_script && travis lint
     open ./images/grid_console.png && open ./videos/chrome/*.mkv
-    git checkout -b tmp-2.53.1x && git checkout ./images/grid_console.png
+    git checkout -b tmp-2.53.1y && git checkout ./images/grid_console.png
     #git add ... git commit ... git push ... open pull request
 
 For repository owners only:
 
-    git commit -m "Upgrade Chrome stable patch 53.0.2785.116"
-    git tag -d latest && git tag 2.53.1w && git push origin tmp-2.53.1x && git push --tags
+    git commit -m "geckodriver, Ubuntu 20160923, chromedriver.log, DBUS magic"
+    git tag -d latest && git tag 2.53.1y && git push origin tmp-2.53.1y && git push --tags
 
 -- Wait for Travis to pass OK
 -- Make sure changes got merged into master by elgalubot
 
-    git checkout master && git pull && git branch -d tmp-2.53.1x && git push origin --delete tmp-2.53.1x
+    git checkout master && git pull && git branch -d tmp-2.53.1y && git push origin --delete tmp-2.53.1y
 
 -- Re-add TBD_* section in CHANGELOG.md starting with TBD_DOCKER_TAG
 -- If Chrome version changed upload:
@@ -27,18 +27,17 @@ For repository owners only:
 ### Chrome artifact
 Keep certain bins if chrome version changed for example:
 
-    cd ~/tmp_binaries && VER="53.0.2785.116"
-    NAME="google-chrome-stable_${VER}_amd64" && echo ${NAME}
+    cd ~/tmp_binaries && VER="53.0.2785.116" && NAME="google-chrome-stable_${VER}_amd64" && echo ${NAME}
     wget -nv --show-progress -O ${NAME}.deb "https://dl.google.com/linux/direct/google-chrome-stable_current_amd64.deb"
     md5sum ${NAME}.deb > ${NAME}.md5 && shasum ${NAME}.deb > ${NAME}.sha
     cp ${NAME}.md5 ${NAME}.sha ~/docker/binaries
 
 ## Retry
 Failed in Travis? retry
 
-    git tag -d 2.53.1w && git push origin :2.53.1w
+    git tag -d 2.53.1y && git push origin :2.53.1y
     #git add ...
-    git commit --amend && git tag 2.53.1w && git push --force origin tmp-2.53.1x && git push --tags
+    git commit --amend && git tag 2.53.1y && git push --force origin tmp-2.53.1y && git push --tags
 
 ## Docker push from Travis CI
 Travis [steps](https://docs.travis-ci.com/user/docker/#Pushing-a-Docker-Image-to-a-Registry) involve `docker login` and docker credentials encryptions.

Dockerfile

@@ -3,9 +3,9 @@
 ###################################################
 #== Ubuntu xenial is 16.04, i.e. FROM ubuntu:16.04
 # search for more at https://registry.hub.docker.com/_/ubuntu/tags/manage/
-FROM ubuntu:xenial-20160818
+FROM ubuntu:xenial-20160923.1
 ENV UBUNTU_FLAVOR="xenial" \
-    UBUNTU_DATE="20160818"
+    UBUNTU_DATE="20160923.1"
 
 #== Ubuntu wily is 15.10, i.e. FROM ubuntu:15.10
 # FROM ubuntu:wily-20151208
@@ -47,6 +47,9 @@ RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 40976EAF437D05B5 \
 # Remove with: sudo apt-key del 3B4FE6ACC0B21F32
 RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys 3B4FE6ACC0B21F32 \
   && apt-key update -qqy
+# Remove with: sudo apt-key del A2F683C52980AECF
+RUN apt-key adv --keyserver keyserver.ubuntu.com --recv-keys A2F683C52980AECF \
+  && apt-key update -qqy
 
 #========================
 # Miscellaneous packages
@@ -418,16 +421,6 @@ RUN apt-get -qqy update \
 #   && npm install -g gifify \
 #   && gifify --version
 
-#==========================
-# Mozilla Firefox - Latest
-#==========================
-# RUN apt-get -qqy update \
-#   && apt-get -qqy install \
-#     firefox \
-#   && rm -rf /var/lib/apt/lists/*
-# If we are not installing latest from apt sources then remove (purge) default
-RUN apt-get -qqy purge firefox
-
 #=========================================================
 # Python2 for Supervisor, selenium tests, and other stuff
 #=========================================================
@@ -460,6 +453,7 @@ RUN apt-get -qqy update \
 
 ENV CPU_ARCH 64
 ENV SEL_HOME ${NORMAL_USER_HOME}/selenium
+RUN mkdir -p ${SEL_HOME}
 
 #===============================
 # Mozilla Firefox install tools
@@ -673,12 +667,32 @@ RUN cd /tmp \
 #       && rm firefox-${FF_VER}.${FF_LANG}.linux64.tar.bz2 \
 #      ;done
 
+#==========================
+# Mozilla Firefox - Latest
+#==========================
+# RUN apt-get -qqy update \
+#   && apt-get -qqy install \
+#     firefox \
+#   && rm -rf /var/lib/apt/lists/*
+# ENV FF_VER="latest" \
+#     FF_LANG="en-US" \
+#     FF_DEST="/usr/lib/firefox"
+# ENV FIREFOX_DEST_BIN="${FF_DEST}/firefox"
+# RUN ln -fs ${FIREFOX_DEST_BIN} /usr/bin/firefox
+
 #-------------------------#
 # FIREFOX_VERSIONS Latest #
 #-------------------------#
 # Install Latest available firefox version
 # this also used to work: ENV FIREFOX_LATEST_VERSION latest
 #
+#--- Nightly
+# ENV FF_VER="52.0a0" \
+#     FF_PLATFORM="linux-i686" \
+#     FF_INNER_PATH="firefox/nightly/latest-mozilla-central"
+# ENV FF_COMP="firefox-${FF_VER}.${FF_LANG}.${FF_PLATFORM}.tar.bz2"
+# ENV FF_URL="${FF_BASE_URL}/${FF_INNER_PATH}/${FF_COMP}"
+#
 # Where to find latest version:
 #  https://archive.mozilla.org/pub/mozilla.org/firefox/releases/latest/linux-x86_64/en-US/
 #  https://download-installer.cdn.mozilla.net/pub/mozilla.org/firefox/releases/latest/linux-x86_64/en-US/
@@ -688,19 +702,42 @@ RUN cd /tmp \
 # FF_LANG can be either en-US // de // fr and so on
 # Regarding the pip packages, see released versions at:
 #  https://github.com/mozilla/mozdownload/releases
-ENV FF_VER="47.0.1" \
-    FF_LANG="en-US" \
-    FF_PLATFORM="linux-x86_64" \
+
+ENV FF_LANG="en-US" \
     FF_BASE_URL="https://archive.mozilla.org/pub" \
     FF_DEST="${SEL_HOME}/firefox"
-ENV FF_COMP="firefox-${FF_VER}.tar.bz2" \
-    FIREFOX_DEST_BIN="${FF_DEST}/firefox"
-ENV FF_URL "${FF_BASE_URL}/firefox/releases/${FF_VER}/${FF_PLATFORM}/${FF_LANG}/${FF_COMP}"
+ENV FIREFOX_DEST_BIN="${FF_DEST}/firefox"
+
+#--- Stable
+ENV FF_VER="47.0.1" \
+    FF_PLATFORM="linux-x86_64" \
+    FF_INNER_PATH="firefox/releases"
+ENV FF_COMP="firefox-${FF_VER}.tar.bz2"
+ENV FF_URL="${FF_BASE_URL}/${FF_INNER_PATH}/${FF_VER}/${FF_PLATFORM}/${FF_LANG}/${FF_COMP}"
+
 RUN mkdir -p ${SEL_HOME} && cd ${SEL_HOME} \
   && wget -nv "${FF_URL}" -O "firefox.tar.bz2" \
   && bzip2 -d "firefox.tar.bz2" \
   && tar xf "firefox.tar" \
-  && rm "firefox.tar"
+  && rm "firefox.tar" \
+  && sudo ln -fs ${FIREFOX_DEST_BIN} /usr/bin/firefox
+
+#============
+# GeckoDriver
+#============
+ENV GECKOD_VER="0.10.0" \
+    GECKOD_URL="https://github.com/mozilla/geckodriver/releases/download"
+RUN wget --no-verbose -O /tmp/geckodriver.tar.gz \
+     "${GECKOD_URL}/v${GECKOD_VER}/geckodriver-v${GECKOD_VER}-linux64.tar.gz" \
+  && rm -rf /opt/geckodriver* \
+  && tar -C /opt -xvzf /tmp/geckodriver.tar.gz \
+  && mv /opt/geckodriver /usr/bin/geckodriver \
+  && chmod +x /usr/bin/geckodriver \
+  && ln -fs /usr/bin/geckodriver /opt/geckodriver \
+  && ln -fs /usr/bin/geckodriver ${FF_DEST}/geckodriver \
+  && ln -fs /usr/bin/geckodriver /usr/bin/wires \
+  && ln -fs /usr/bin/geckodriver ${FF_DEST}/wires \
+  && rm /tmp/geckodriver.tar.gz
 
   # && rm -rf ${NORMAL_USER_HOME}/firefox-src
 # RUN mkdir -p ${NORMAL_USER_HOME}/firefox-src \
@@ -977,6 +1014,8 @@ ENV FIREFOX_VERSION="${FF_VER}" \
   SELENIUM_NODE_PROXY_PARAMS="" \
   # To taggle issue #58 see https://goo.gl/fz6RTu
   CHROME_ARGS="--no-sandbox" \
+  # Will be passed with: -Dwebdriver.chrome.verboseLogging
+  CHROME_VERBOSELOGGING="true" \
   # e.g. CHROME_ARGS="--no-sandbox --ignore-certificate-errors" \
   # SELENIUM_NODE_CHROME_PARAMS='-Dselenium.chrome.args="--no-sandbox"' \
   # WEBDRIVER_NODE_CHROME_PARAMS='-Dwebdriver.chrome.args="--no-sandbox"' \
@@ -1125,9 +1164,8 @@ ENV FIREFOX_VERSION="${FF_VER}" \
   # Usage: docker run -v /var/run/docker.sock:/var/run/docker.sock
   #                   -v $(which docker):$(which docker)
   DOCKER_SOCK="/var/run/docker.sock" \
-  # DBUS hack thanks @pwaller
-  #  https://github.com/SeleniumHQ/docker-selenium/issues/87#issuecomment-187659234
-  DBUS_SESSION_BUS_ADDRESS=/dev/null \
+  # for DBUS hacks, see entry.sh
+  # DBUS_SESSION_BUS_ADDRESS="unix:abstract=/dev/null" \
   # Selenium test steps sleep
   TEST_SLEEPS="0.5" \
   # Restore

README.md

@@ -256,14 +256,14 @@ If the VNC password was randomly generated find out with
 
 ### Chrome crashed
 
-If your tests crashes in Chrome you may need to increase shm size or simply start your container by sharing `-v /dev/shm:/dev/shm`
+If your tests crashes in Chrome you may need to increase shm size or simply start your container by sharing `-v /dev/shm:/dev/shm` or, alternatively, `--shm-size=1g`
 
     docker run ... -v /dev/shm:/dev/shm
 
 Alternatively you can increase it inside the container:
 
 1. start docker in privileged mode: `docker run --privileged`
-2. increase shm size from default 64mb to something bigger:
+2. increase shm size from default 64mb or 68mb to something bigger:
 
 ```sh
 docker exec grid sudo umount /dev/shm

TBD_DOCKER_TAG.md

@@ -16,7 +16,7 @@
   + Tested on kernel CI  host: TBD_HOST_UNAME
   + Built at dev host with: Docker version 1.12.1, build 23cf638
   + Built at CI  host with: Docker version TBD_DOCKER_VERS, build TBD_DOCKER_BUILD
-  + Built at dev host with: Docker Compose version 1.8.0, build f3628c7
+  + Built at dev host with: Docker Compose version 1.8.1, build 878cff1
   + Built at CI  host with: Docker Compose version TBD_DOCKER_COMPOSE_VERS, build TBD_DOCKER_COMPOSE_BUILD
   + Image size: TBD_IMAGE_SIZE
   + Digest: TBD_DIGEST

bin/entry.sh

@@ -1,5 +1,24 @@
 #!/usr/bin/env bash
 
+# set -e: exit asap if a command exits with a non-zero status
+set -e
+
+# Workaround that might help to get dbus working in docker
+#  http://stackoverflow.com/a/38355729/511069
+#  https://github.com/SeleniumHQ/docker-selenium/issues/87#issuecomment-187659234
+#  - still unclear if this helps: `-v /var/run/dbus:/var/run/dbus`
+#  - this works generates errors: DBUS_SESSION_BUS_ADDRESS="/dev/null"
+#  - this gives less erros: DBUS_SESSION_BUS_ADDRESS="unix:abstract=/dev/null"
+sudo rm -f /var/lib/dbus/machine-id
+sudo mkdir -p /var/run/dbus
+sudo service dbus restart
+# Test dbus works
+service dbus status
+export $(dbus-launch)
+export NSS_USE_SHARED_DB=ENABLED
+echo "-- INFO: DBUS_SESSION_BUS_ADDRESS=${DBUS_SESSION_BUS_ADDRESS}"
+echo "-- INFO: DBUS_SESSION_BUS_PID=${DBUS_SESSION_BUS_PID}"
+
 #-----------------------------------------------
 # Perform cleanup to support `docker restart`
 stop 2>&1 >/dev/null || true

docker-compose-host.yml

@@ -15,6 +15,8 @@ services:
     image: elgalu/selenium
     network_mode: host
     shm_size: 1g
+    # volumes:
+    #   - /var/run/dbus:/var/run/dbus
     environment:
       - SELENIUM_HUB_PORT
       - PICK_ALL_RANDMON_PORTS=true

docker-compose-jenkins.yml

@@ -26,6 +26,8 @@ services:
     memswap_limit: 4g
     # 90000 means 90%
     # cpu_quota: 90000
+    # volumes:
+    #   - /var/run/dbus:/var/run/dbus
     environment:
       - SELENIUM_HUB_PORT
       - VNC_FROM_PORT