allow testing in Selenium locally using --selenium

nolanlawson · nolanlawson · commit cb0c85a433df · 2016-03-19T18:35:55.000-04:00
diff --git a/README.md b/README.md
@@ -24,6 +24,26 @@ When iterating on your tests during development, simply use zuul `--local` mode
 
 See the [quickstart](https://github.com/defunctzombie/zuul/wiki/quickstart) page on the wiki for more details.
 
+### Automated browser tests
+
+You can test in PhantomJS using:
+
+    zuul --phantom
+
+Note that PhantomJS must be installed separately, e.g. `npm install phantomjs`.
+
+You can also test using Selenium against any browser you have installed locally. For instance:
+
+    zuul --selenium
+
+will test in the default browser (Firefox). To test in another browser, you can do:
+
+    zuul --selenium --browser-name chrome
+
+Or:
+
+    zuul --selenium --browser-name firefox --browser-version 41.0.1
+
 ### Cross browser testing via Saucelabs
 
 The reason we go through all this trouble in the first place is to seamlessly run our tests against all those browsers we don't have installed. Luckily, [saucelabs](https://saucelabs.com/) runs some browsers and we can easily task zuul to test on those.
diff --git a/bin/zuul b/bin/zuul
@@ -25,6 +25,7 @@ program
 .option('--phantom-remote-debugger-port [port]', 'connect phantom to remote debugger')
 .option('--phantom-remote-debugger-autorun', 'run tests automatically when --phantom-remote-debugger-port is specified')
 .option('--electron', 'run tests in electron. electron must be installed separately.')
+.option('--selenium', 'run tests in Selenium, locally. Default browser: Chrome. Use --browser-name and --browser-version for others')
 .option('--tunnel-host <host url>', 'specify a localtunnel server to use for forwarding')
 .option('--sauce-connect [tunnel-identifier]', 'use saucelabs with sauce connect instead of localtunnel. Optionally specify the tunnel-identifier')
 .option('--server <the server script>', 'specify a server script to be run')
@@ -48,6 +49,7 @@ var config = {
     phantomRemoteDebuggerPort: program.phantomRemoteDebuggerPort,
     phantomRemoteDebuggerAutorun: program.phantomRemoteDebuggerAutorun,
     electron: program.electron,
+    selenium: program.selenium,
     prj_dir: process.cwd(),
     tunnel_host: program.tunnelHost,
     sauce_connect: program.sauceConnect,
@@ -127,12 +129,14 @@ if (config.files.length === 0) {
     return process.exit(1);
 }
 
-if ((program.browserVersion || program.browserPlatform) && !program.browserName) {
+if (((program.browserVersion || program.browserPlatform) && !program.browserName) &&
+    !program.selenium) {
     console.error('the browser name needs to be specified (via --browser-name)');
     return process.exit(1);
 }
 
-if ((program.browserName || program.browserPlatform) && !program.browserVersion) {
+if (((program.browserName || program.browserPlatform) && !program.browserVersion) &&
+    !program.selenium) {
     console.error('the browser version needs to be specified (via --browser-version)');
     return process.exit(1);
 }
@@ -188,7 +192,7 @@ if (config.local) {
     return zuul.run(function(passed) {
     });
 }
-else if (config.phantom || config.electron) {
+else if (config.phantom || config.electron || config.selenium) {
     return zuul.run(function(passed) {
         process.exit(passed ? 0 : 1);
     });
diff --git a/lib/Selenium.js b/lib/Selenium.js
@@ -0,0 +1,187 @@
+var path = require('path');
+var EventEmitter = require('events').EventEmitter;
+var debug = require('debug')('zuul:selenium');
+var wd = require('wd');
+
+var SELENIUM_VERSION = '2.52.0';
+
+var setup_test_instance = require('./setup');
+require('colors');
+
+function Selenium(opt) {
+    if (!(this instanceof Selenium)) {
+        return new Selenium(opt);
+    }
+
+    var self = this;
+    self._opt = opt;
+    self._browserName = (opt.browsers && opt.browsers[0] && opt.browsers[0].name) || null;
+    self._browserVersion = (opt.browsers && opt.browsers[0] && opt.browsers[0].version) || null;
+    self.status = {
+        passed: 0,
+        failed: 0
+    };
+}
+
+Selenium.prototype.__proto__ = EventEmitter.prototype;
+
+Selenium.prototype.start = function() {
+    var self = this;
+
+    var seleniumClient;
+    var selenium = require('selenium-standalone');
+
+    function finish() {
+        if (self._finished) {
+            return;
+        }
+        self._finished = true;
+        reporter.removeAllListeners();
+        if (seleniumClient) {
+            seleniumClient.quit();
+        }
+    }
+
+    self.controller = setup_test_instance(self._opt, function(err, url) {
+        if (err) {
+            console.log('Error: %s'.red, err);
+            self.emit('done', {
+                passed: false
+            });
+            finish();
+        }
+
+        debug('url %s', url);
+
+        var reporter = new EventEmitter();
+
+        reporter.on('console', function(msg) {
+            console.log.apply(console, msg.args);
+        });
+
+        reporter.on('test', function(test) {
+            console.log('starting', test.name.white);
+        });
+
+        reporter.on('test_end', function(test) {
+            if (!test.passed) {
+                console.log('failed', test.name.red);
+                return self.status.failed++;
+            }
+
+            console.log('passed:', test.name.green);
+            self.status.passed++;
+        });
+
+        reporter.on('assertion', function(assertion) {
+            console.log('Error: %s'.red, assertion.message);
+            assertion.frames.forEach(function(frame) {
+                console.log('    %s %s:%d'.grey, frame.func, frame.filename, frame.line);
+            });
+            console.log();
+        });
+
+        reporter.on('done', function() {
+            finish();
+        });
+
+        self.emit('init', url);
+        self.emit('start', reporter);
+
+        var opts = {version: SELENIUM_VERSION};
+        selenium.install(opts, function(err) {
+            if (err) {
+                console.log('Error: %s'.red, new Error(
+                    'Failed to install selenium'));
+                self.emit('done', {
+                    passed: false
+                });
+                finish();
+                return;
+            }
+            selenium.start(opts, function() {
+                seleniumClient = wd.promiseChainRemote();
+                onSeleniumReady();
+            });
+        });
+
+        function onSeleniumClientReady() {
+            var lastMessage = Date.now();
+            var script = 'window.zuul_msg_bus ? ' +
+                'window.zuul_msg_bus.splice(0, window.zuul_msg_bus.length) : ' +
+                '[]';
+
+            var interval = setInterval(poll, 2000);
+
+            function onDoneMessage() {
+                clearInterval(interval);
+                seleniumClient.quit(function () {
+                    seleniumClient = null;
+                    self.emit('done', {
+                        passed: self.status.passed,
+                        failed: self.status.failed
+                    });
+                    finish();
+                });
+            }
+
+            function onGetMessages(err, messages) {
+                if (err) {
+                    self.emit('error', err);
+                } else if (messages.length) {
+                    lastMessage = Date.now();
+                    messages.forEach(function (msg) {
+                        debug('msg: %j', msg);
+                        if (msg.type === 'done') {
+                            onDoneMessage();
+                        } else {
+                            reporter.emit(msg.type, msg);
+                        }
+                    });
+                } else if ((Date.now() - lastMessage) > testTimeout) {
+                    clearInterval(interval);
+                    console.log('Error: %s'.red, new Error(
+                        'selenium timeout after ' + testTimeout + ' ms'));
+                    self.emit('done', {
+                        passed: false
+                    });
+                    finish();
+                }
+            }
+
+            function poll() {
+                seleniumClient.eval(script, onGetMessages);
+            }
+        }
+
+        function onSeleniumReady() {
+            // TODO: maybe these should be configurable
+            var testTimeout = 120000;
+            var tunnelId = process.env.TRAVIS_JOB_NUMBER || 'tunnel-' + Date.now();
+            var opts = {
+                tunnelTimeout: testTimeout,
+                name: self._browserName + ' - ' + tunnelId,
+                'max-duration': 60 * 45,
+                'command-timeout': 599,
+                'idle-timeout': 599,
+                'tunnel-identifier': tunnelId
+            };
+            if (self._browserName) {
+                opts.browserName = self._browserName;
+            }
+            if (self._browserVersion) {
+                opts.version = self._browserVersion;
+            }
+
+            seleniumClient.init(opts).get(url, onSeleniumClientReady);
+        }
+    });
+};
+
+Selenium.prototype.shutdown = function() {
+    if (self.controller) {
+        self.controller.shutdown();
+    }
+};
+
+module.exports = Selenium;
diff --git a/lib/zuul.js b/lib/zuul.js
@@ -10,6 +10,7 @@ var setup_test_instance = require('./setup');
 var SauceBrowser = require('./SauceBrowser');
 var PhantomBrowser = require('./PhantomBrowser');
 var Electron = require('./Electron');
+var Selenium = require('./Selenium');
 
 module.exports = Zuul;
 
@@ -125,6 +126,14 @@ Zuul.prototype.run = function(done) {
             });
             return electron.start();
         }
+        if (config.selenium) {
+            var selenium = Selenium(config);
+            self.emit('browser', selenium);
+            selenium.once('done', function(results) {
+                done(results.failed === 0 && results.passed > 0);
+            });
+            return selenium.start();
+        }
 
         var batch = new Batch();
         batch.concurrency(self._concurrency);
diff --git a/package.json b/package.json
@@ -28,6 +28,7 @@
     "lodash": "3.10.1",
     "opener": "1.4.0",
     "osenv": "0.0.3",
+    "selenium-standalone": "^5.0.0",
     "shallow-copy": "0.0.1",
     "shell-quote": "1.4.1",
     "stack-mapper": "0.2.2",
@@ -36,7 +37,7 @@
     "tap-finished": "0.0.1",
     "tap-parser": "0.7.0",
     "watchify": "3.7.0",
-    "wd": "0.3.11",
+    "wd": "0.4.0",
     "xtend": "2.1.2",
     "yamljs": "0.1.4",
     "zuul-localtunnel": "1.1.0"
@@ -70,4 +71,4 @@
   "scripts": {
     "test": "DEBUG=zuul* mocha --ui qunit --timeout 0 --bail -- test/index.js"
   }
-}
+}
diff --git a/test/integration/jamine-selenium.js b/test/integration/jamine-selenium.js
@@ -0,0 +1,36 @@
+var Zuul = require('../../');
+
+var after = require('after');
+var assert = require('assert');
+
+test('jasmine - phantom', function(done) {
+    done = after(3, done);
+
+    var config = {
+        ui: 'jasmine',
+        prj_dir: __dirname + '/../fixtures/jasmine',
+        selenium: true,
+        concurrency: 1,
+        files: [__dirname + '/../fixtures/jasmine/test.js']
+    };
+
+    var zuul = Zuul(config);
+
+    // each browser we test will emit as a browser
+    zuul.on('browser', function(browser) {
+        browser.on('init', function() {
+            done();
+        });
+
+        browser.on('done', function(results) {
+            assert.equal(results.passed, 1);
+            assert.equal(results.failed, 1);
+            done();
+        });
+    });
+
+    zuul.run(function(passed) {
+        assert.ok(!passed);
+        done();
+    });
+});
diff --git a/test/integration/mocha-qunit-selenium.js b/test/integration/mocha-qunit-selenium.js
@@ -0,0 +1,39 @@
+var Zuul = require('../../');
+
+var after = require('after');
+var assert = require('assert');
+
+test('mocha-qunit - phantom', function(done) {
+    done = after(3, done);
+
+    var config = {
+        ui: 'mocha-qunit',
+        prj_dir: __dirname + '/../fixtures/mocha-qunit',
+        selenium: true,
+        concurrency: 1,
+        files: [__dirname + '/../fixtures/mocha-qunit/test.js']
+    };
+    var zuul = Zuul(config);
+
+    zuul.on('browser', function(browser) {
+        browser.once('start', function(reporter) {
+            reporter.once('done', function(results) {
+                assert.equal(results.passed, false);
+                assert.equal(results.stats.passed, 1);
+                assert.equal(results.stats.failed, 1);
+                done();
+            });
+        });
+
+        browser.on('done', function(results) {
+            assert.equal(results.passed, 1);
+            assert.equal(results.failed, 1);
+            done();
+        });
+    });
+
+    zuul.run(function(passed) {
+        assert.ok(!passed);
+        done();
+    });
+});
diff --git a/test/integration/mochabdd-selenium.js b/test/integration/mochabdd-selenium.js
diff --git a/test/integration/tape-selenium.js b/test/integration/tape-selenium.js
