Merge pull request #2 from cloudogu/feature/extendWithDockerNetwork

Feature/extend with docker network

Author: mbehlendorf

README.md

@@ -45,20 +45,38 @@ Even more convenient: You could watch the videos in the browser directly. This i
   * Or you start your Jenkins instance with   
   `-Dhudson.model.DirectoryBrowserSupport.CSP="sandbox; default-src 'none'; img-src 'self'; style-src 'self'; media-src 'self';"`
 
+## Docker Network creation
+
+It is possible (although not necessary) to explicitly work with docker networks. This library supports the automatic creation and removal of a bridge network with a unique name.
+
+### How
+
+`withZalenium` accepts now an optional network name the Zalenium container can attach to the given network. Conveniently a docker network can be created with this pipeline step which provides the dynamically created network name.
+
+```
+    withDockerNetwork { networkName ->
+        def yourConfig = [:]
+        withZalenium(yourConfig, networkName) {}
+            docker.image("foo/bar:1.2.3").withRun("--network ${network}") {
+            ...
+    }
+
+```
+
 ## Locking
 
 Right now, only one Job can run Zalenium Tests at a time.
 This could be improved in the future. 
 
-## Why?
+### Why?
 
 When multiple jobs executed we faced non-deterministic issues that the zalenium container was gone all of a sudden, 
 connections were aborted or timed out.
 
 So we implemented a lock before starting zalenium that can only be passed by one job at a time.
 It feels like this issue is gone now, but we're not sure if the lock was the proper fix.
 
-## How?
+### How?
 
 We use the `lock` step of the [Lockable Resources Plugin](https://wiki.jenkins.io/display/JENKINS/Lockable+Resources+Plugin).
 

vars/withDockerNetwork.groovy

@@ -0,0 +1,29 @@
+/**
+ * Start a temporary docker network so docker containers can interact even without IP address. The created network will
+ * be removed automatically once the body finishes.
+ * @param printDebugOutput logs creation and removal of the network if set to true. Can be left out.
+ * @param inner the body to be executed
+  */
+void call(printDebugOutput = false, Closure inner) {
+    def networkName = "net_" + generateJobName()
+
+    try {
+        debugOut(printDebugOutput, "create docker bridge network")
+        sh "docker network create ${networkName}"
+        // provide network name to closure
+        inner.call(networkName)
+    } finally {
+        debugOut(printDebugOutput, "remove docker network")
+        sh "docker network rm ${networkName}"
+    }
+}
+
+void debugOut(boolean printDebugOutput, String logMessage) {
+    if (printDebugOutput) {
+        echo "DEBUG: " + logMessage
+    }
+}
+
+String generateJobName() {
+    return "${JOB_BASE_NAME}_${BUILD_NUMBER}"
+}

vars/withDockerNetwork.txt

@@ -0,0 +1,22 @@
+Starts a temporary docker network so docker containers can interact even without IP address. The created network will
+be removed once the body finishes.
+
+Requires Docker!
+
+(optional) Parameters:
+
+- printDebugOutput - this bool adds network creation and removal output to the Jenkins console log for debugging
+    purposes.
+
+Exemplary calls:
+
+- withDockerNetwork { networkName ->
+    // create your container with the networkName along the lines of this:
+    docker.image("foo/bar:1.2.3").withRun("--network ${networkName}") {
+        ...
+    }
+- debugOutput = true; withDockerNetwork(debugOutput) { networkName ->
+  // prints the creation and removal of the docker network in your output
+          // create your container with the networkName along the lines of this:
+          docker.image("foo/bar:1.2.3").withRun("--network ${networkName}") {
+  }

vars/withZalenium.groovy

@@ -1,45 +1,78 @@
-
-void call(config = [:], Closure closure) {
-
-    def defaultConfig = [seleniumVersion : '3.141.59-p8',
-                         zaleniumVersion : '3.141.59g',
-                         zaleniumVideoDir: "zalenium",
-                         debugZalenium   : false]
+/**
+ * Starts Zalenium in a Selenium grid and executes the given body. When the body finishes, the Zalenium container will
+ * gracefully shutdown and archive all videos generated by the tests.
+ *
+ * @param config contains a map of settings that change the Zalenium behavior. Can be a partial map or even left out.
+ *      The defaults are:
+ *      [seleniumVersion   : '3.141.59-p8',
+ *      seleniumImage      : 'elgalu/selenium',
+ *      zaleniumVersion    : '3.141.59g',
+ *      zaleniumImage      : 'dosel/zalenium',
+ *      zaleniumVideoDir   : 'zalenium',
+ *      sendGoogleAnalytics: false,
+ *      debugZalenium      : false]
+ * @param zaleniumNetwork The Zalenium container will be added to this docker network. This is useful if other containers
+ *      must communicate with Zalenium while being in a docker network. If empty or left out, Zalenium will stay in the
+ *      default network.
+ * @param closure the body
+ */
+void call(Map config = [:], String zaleniumNetwork, Closure closure) {
+
+    def defaultConfig = [seleniumVersion    : '3.141.59-p8',
+                         seleniumImage      : 'elgalu/selenium',
+                         zaleniumVersion    : '3.141.59g',
+                         zaleniumImage      : 'dosel/zalenium',
+                         zaleniumVideoDir   : 'zalenium',
+                         debugZalenium      : false,
+                         sendGoogleAnalytics: false]
 
     // Merge default config with the one passed as parameter
     config = defaultConfig << config
 
     sh "mkdir -p ${config.zaleniumVideoDir}"
 
-    docker.image("elgalu/selenium:${config.seleniumVersion}").pull()
-    def zaleniumImage = docker.image("dosel/zalenium:${config.zaleniumVersion}")
+    // explicitly pull the image into the registry. The documentation is not fully clear but it seems that pull()
+    // will persist the image in the registry better than an docker.image(...).runWith()
+    docker.image("${config.seleniumImage}:${config.seleniumVersion}").pull()
+    def zaleniumImage = docker.image("${config.zaleniumImage}:${config.zaleniumVersion}")
     zaleniumImage.pull()
 
-    uid = findUid()
-    gid = findGid()
+    def uid = findUid()
+    def gid = findGid()
+
+    networkParameter = ""
+
+    if (zaleniumNetwork != null && !zaleniumNetwork.isEmpty()) {
+        networkParameter = "--network ${zaleniumNetwork}"
+    }
 
     lock("zalenium") {
         zaleniumImage.withRun(
                 // Run with Jenkins user, so the files created in the workspace by zalenium can be deleted later
+                // Otherwise that would be root, and you know how hard it is to get rid of root-owned files.
                 "-u ${uid}:${gid} -e HOST_UID=${uid} -e HOST_GID=${gid} " +
-                // Zalenium starts headless browsers in docker containers, so it needs the socket
-                '-v /var/run/docker.sock:/var/run/docker.sock ' +
-                "-v ${WORKSPACE}/${config.zaleniumVideoDir}:/home/seluser/videos",
+                        // Zalenium starts headless browsers in each in a docker container, so it needs the Socket
+                        '-v /var/run/docker.sock:/var/run/docker.sock ' +
+                        '--privileged ' +
+                        "${networkParameter} " +
+                        "-v ${WORKSPACE}/${config.zaleniumVideoDir}:/home/seluser/videos",
                 'start ' +
-                        "${config.debugZalenium ? '--debugEnabled true' : ''}"
+                        "--seleniumImageName ${config.seleniumImage} " +
+                        "${config.debugZalenium ? '--debugEnabled true' : ''} " +
+                        // switch off analytic gathering
+                        "${config.sendGoogleAnalytics ? '--sendAnonymousUsageInfo false' : ''} "
         ) { zaleniumContainer ->
-
-            def zaleniumIp = findContainerIp(zaleniumContainer)
-
-            waitForSeleniumToGetReady(zaleniumIp)
-            // Delete videos from previous builds, if any
-            // This also works around the bug that zalenium stores files as root (before version 3.141.59f)
-            // https://github.com/zalando/zalenium/issues/760
-            // This workaround still leaves a couple of files owned by root in the zaleniumVideoDir
-            resetZalenium(zaleniumIp)
+            String zaleniumIp = findContainerIp(zaleniumContainer)
 
             try {
-                closure(zaleniumIp)
+                waitForSeleniumToGetReady(zaleniumIp)
+                // Delete videos from previous builds, if any
+                // This also works around the bug that zalenium stores files as root (before version 3.141.59f)
+                // https://github.com/zalando/zalenium/issues/760
+                // This workaround still leaves a couple of files owned by root in the zaleniumVideoDir
+                resetZalenium(zaleniumIp)
+
+                closure.call(zaleniumContainer, zaleniumIp, uid, gid)
             } finally {
                 // Wait for Selenium sessions to end (i.e. videos to be copied)
                 // Leaving the withRun() closure leads to "docker rm -f" being called, cancelling copying
@@ -52,23 +85,23 @@ void call(config = [:], Closure closure) {
                 sh "docker logs ${zaleniumContainer.id} > zalenium-docker.log 2>&1"
             }
         }
-
     }
 }
 
 String findContainerIp(container) {
-    sh (returnStdout: true,
+    sh(returnStdout: true,
             script: "docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' ${container.id}")
             .trim()
 }
 
 String findUid() {
-    sh (returnStdout: true,
+    sh(returnStdout: true,
             script: 'id -u')
             .trim()
 }
+
 String findGid() {
-    sh (returnStdout: true,
+    sh(returnStdout: true,
             script: 'id -g')
             .trim()
 }
@@ -107,4 +140,4 @@ boolean isSeleniumSessionsActive(String host) {
 void resetZalenium(String host) {
     sh(returnStatus: true,
             script: "curl -sSL http://${host}:4444/dashboard/cleanup?action=doReset") == 0
-}
+}

vars/withZalenium.txt

@@ -1,19 +1,22 @@
-Starts a temporary zalenium server, that stores videos of the selenium tests in the workspace.
+Starts a temporary Zalenium server that stores videos of the selenium tests in the workspace.
 
 Requires Docker!
 
 (optional) Parameters:
 
-- seleniumVersion - version of the "elgalu/selenium" docker image
-- zaleniumVersion - version of the "dosel/zalenium" docker image
-- zaleniumVideoDir - workspace relative path where the videos are stored
+- seleniumImage - the full name of the selenium image name including the registry. Defaults to 'elgalu/selenium' from hub.docker.com. The selenium image is used by Zalenium.
+- seleniumVersion - version of the selenium docker image
+- zaleniumImage - the full name of the zalenium image name including the registry. Defaults to 'dosel/zalenium' from hub.docker.com.
+- zaleniumVersion - version of the zalenium docker image
+- zaleniumVideoDir - path where the videos are stored, relative to the Jenkins workspace.
 - debugZalenium - makes the zalenium container write a lot more logs
+- sendGoogleAnalytics - if true this will send analytic data to Google/Zalando
 
 Exemplary calls:
 
-- withZalenium { zaleniumIp ->
+- withZalenium { zaleniumContainer, zaleniumIp, userid, groupid ->
     // call your selenium tests using maven, yarn, etc.
   }
-- withZalenium([ seleniumVersion : '3.14.0-p15' ]) { zaleniumIp ->
+- withZalenium([ seleniumVersion : '3.14.0-p15' ]) { zaleniumContainer, zaleniumIp, userid, groupid ->
       // call your selenium tests using maven, yarn, etc.
   }