Merge pull request #22 from codebendercc/seleniumbender-improvements

Seleniumbender improvements

Author: dastergon

.gitignore

@@ -1,6 +1,7 @@
 __pycache__/
 .cache/
 bin/env_vars.sh
+bin/config.cfg
 .tox
 *.pyc
 *.swp

README.md

@@ -114,7 +114,7 @@ The most important class is `SeleniumTestCase`, which all of the unit test cases
 inherit from. This grants them access (via `self`) to a number of methods and
 attributes that are useful for performing codebender-specific actions.
 
-**`codebender_testing/capabilities.yaml`** defines a list of `capabilities` to
+**`codebender_testing/capabilities_{firefox, chrome}.yaml`** defines a list of `capabilities` to
 be passed as arguments when instantiating remote webdrivers. In particular, it
 specifies the web browsers that we would like to use. Consult this file for more
 information.
@@ -141,4 +141,3 @@ compile.
 The `logs/` directory contains the results of running certain tests, e.g.
 whether certain sets of sketches have compiled successfully (see "Compilation
 Logs").
-

bin/config.cfg.template

@@ -0,0 +1,31 @@
+[common]
+CAPABILITIES: capabilities_firefox.yaml
+SAUCELABS_USER: ~
+SAUCELABS_KEY: ~
+SAUCELABS_HUB_URL: http://SAUCELABS_USER:[email protected]:80/wd/hub
+LOCAL_HUB_URL: http://127.0.0.1:4444/wd/hub
+DISQUS_SSO_ID: ~
+DISQUS_SSO_USERNAME: ~
+DISQUS_SSO_EMAIL: ~
+CODEBENDER_TEST_USER: ~
+CODEBENDER_TEST_PASS: ~
+EMAIL: ~
+ROOTDIR: ~
+
+[live]
+DISQUS_ACCESS_TOKEN: ~
+DISQUS_API_SECRET: ~
+DISQUS_API_PUBLIC: ~
+DISQUS_FORUM: ~
+AUTHOR_URL: ~
+
+[staging]
+DISQUS_ACCESS_TOKEN: ~
+DISQUS_API_SECRET: ~
+DISQUS_API_PUBLIC: ~
+DISQUS_FORUM: ~
+AUTHOR_URL: ~
+
+[local]
+CODEBENDER_TEST_USER: ~
+CODEBENDER_TEST_PASS: ~

bin/env_vars.sh.template

@@ -1,10 +1,12 @@
 #!/bin/bash
 
-export CAPABILITIES='capabilities_firefox.yaml'
+export CAPABILITIES="capabilities_firefox.yaml"
 
 SAUCELABS_USER=""
 SAUCELABS_KEY=""
-export CODEBENDER_SELENIUM_HUB_URL=http://${SAUCELABS_USER}:${SAUCELABS_KEY}@ondemand.saucelabs.com:80/wd/hub
+export SAUCELABS_HUB_URL="http://${SAUCELABS_USER}:${SAUCELABS_KEY}@ondemand.saucelabs.com:80/wd/hub"
+export LOCAL_HUB_URL="http://127.0.0.1:4444/wd/hub"
+CODEBENDER_SELENIUM_HUB_URL=$LOCAL_HUB_URL
 
 export CODEBENDER_TEST_USER=""
 export CODEBENDER_TEST_PASS=""

bin/seleniumbender.py

@@ -1,19 +1,19 @@
 #!/usr/bin/env python
 
 from __future__ import print_function
+import ConfigParser
 import argparse
 import os
 import re
 import subprocess
 import sys
 import time
+import yaml
 
 SHELL='/bin/bash'
-SOURCE = 'codebender_cc'
 
 class Tests:
     def __init__(self, url, environment):
-        self.source=SOURCE
         self.url = url
         self.environment = os.path.abspath(environment)
         self.email = os.getenv('EMAIL', '[email protected]')
@@ -39,19 +39,17 @@ def run(self, operation, libraries=None):
         elif operation == 'staging':
             self.staging()
 
-    def run_command(self, command, user_agent=None):
-        if user_agent:
-            os.environ['SELENIUM_USER_AGENT'] = user_agent
+    def run_command(self, command):
         command = ' '.join(command)
         print('command:', command)
         return subprocess.call(command, shell=True, executable=SHELL)
 
     def send_mail_no_logs(self, identifier):
         command = ['mail', '-s', '"Selenium Tests: {identifier} Failed To Run" {email} <<< "Something went wrong with {identifier} tests. Please check the logs."'.format(identifier=identifier, email=self.email)]
-        run_command(command)
+        self.run_command(command)
 
     def send_mail_with_logs(self, identifier):
-        default_tests_dir = os.path.normpath(os.path.join(os.getcwd(), '..'))
+        default_tests_dir = os.path.normpath(os.path.join(os.path.dirname(__file__), '..'))
         root_dir = os.getenv('ROOTDIR', default_tests_dir)
 
         logs = os.path.join(root_dir, 'logs')
@@ -74,12 +72,12 @@ def send_mail_with_logs(self, identifier):
                 'uuencode "{reports}/{reportfile}" "{reportfile}")'.format(reports=reports, reportfile=reportfile),
                 '| mail -s "Selenium Tests Report: {identifier} {email_date} Changes: {changes}" {email}'.format(identifier=identifier, email_date=email_date, changes=changes, email=self.email)
             ]
-            run_command(command)
+            self.run_command(command)
         except:
             pass
 
     def create_command(self, test_directory, *extra_arguments):
-        return ['tox', 'tests/' + test_directory, '--', '--url={}'.format(TARGETS[self.url]), '--source={}'.format(SOURCE)] + list(extra_arguments)
+        return ['tox', 'tests/' + test_directory, '--', '--url={}'.format(TARGETS[self.url])] + list(extra_arguments)
 
     def common(self, identifier='common'):
         command = self.create_command('common', '--plugin')
@@ -115,14 +113,13 @@ def noplugin(self, identifier = 'noplugin'):
     def walkthrough(self, identifier='walkthrough'):
         command = self.create_command('walkthrough', '--plugin')
         retvals = []
-        user_agents = [
-            'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0 codebender-selenium',
-            'Mozilla/5.0 (Windows NT 6.1; rv:43.0) Gecko/20100101 Firefox/43.0 codebender-selenium',
-            'Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1; rv:43.0) Gecko/20100101 Firefox/43.0 codebender-selenium'
-        ]
-        for user_agent in user_agents:
-            retval = self.run_command(command, user_agent=user_agent)
+        for platform in USER_AGENTS.keys():
+            for browser in USER_AGENTS[platform].keys():
+                os.environ['SELENIUM_USER_AGENT_' + browser.upper()] = USER_AGENTS[platform][browser]
+            os.environ['SELENIUM_PLATFORM'] = platform
+            retval = self.run_command(command)
             retvals.append(retval)
+
         retval = max(retvals)
         if retval != 0:
             self.send_mail_no_logs(identifier)
@@ -148,6 +145,35 @@ def staging(self):
     'local': 'http://dev.codebender.cc'
 }
 
+PLATFORMS = {
+    'Linux': 'X11; Ubuntu; Linux x86_64',
+    'Windows 7': 'Windows NT 6.1',
+    'OS X 10.11': 'Macintosh; Intel Mac OS X 10_11_1'
+}
+USER_AGENTS = {}
+USER_AGENT_IDENTIFIER = 'codebender-selenium'
+
+def generate_user_agents(file_path):
+    with open(file_path, 'rb') as fp:
+        capabilities_file = yaml.load(fp)
+    for section in capabilities_file:
+        browser = section['browserName']
+        for platform in PLATFORMS.keys():
+            if platform == 'Linux':
+                os.environ['SELENIUM_PLATFORM'] = platform
+            version = section['version']
+            if browser == 'chrome':
+                user_agent = 'Mozilla/5.0 ({platform}) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/{version}.0.2564.109 Safari/537.36 {identifier}'.format(platform=PLATFORMS[platform], version=version, identifier=USER_AGENT_IDENTIFIER)
+                if platform == 'Linux':
+                    os.environ['SELENIUM_USER_AGENT_CHROME'] = user_agent
+            elif browser == 'firefox':
+                user_agent = 'Mozilla/5.0 ({platform}; rv:{version}.0) Gecko/20100101 Firefox/{version}.0 {identifier}'.format(platform=PLATFORMS[platform], version=version, identifier=USER_AGENT_IDENTIFIER)
+                if platform == 'Linux':
+                    os.environ['SELENIUM_USER_AGENT'] = user_agent
+
+            USER_AGENTS.setdefault(platform, {})
+            USER_AGENTS[platform][browser] = user_agent
+
 def main():
     available_operations = ['{operation}\t{description}'.format(operation=x, description=OPERATIONS[x]) for x in sorted(OPERATIONS.keys())]
     available_targets = ['{target}\t{url}'.format(target=x, url=TARGETS[x]) for x in sorted(TARGETS.keys())]
@@ -161,7 +187,7 @@ def main():
                         default='live',
                         help='Target site for the tests.\nAvailable targets (default: live):\n\t{targets}'.format(targets='\n\t'.join(available_targets)))
     parser.add_argument('--config',
-                        default='env_vars.sh',
+                        default='config.cfg',
                         help='Configuration file to load (default: config.cfg).')
     parser.add_argument('--libraries',
                         default=None,
@@ -170,6 +196,9 @@ def main():
                         action='store_true',
                         default=False,
                         help='Use saucelabs as the Selenium server')
+    parser.add_argument('--capabilities',
+                        default='capabilities_firefox.yaml',
+                        help='Selenium capabilities file (default: capabilities_firefox.yaml).'.format())
 
     # Parse arguments
     args = parser.parse_args()
@@ -185,21 +214,47 @@ def main():
         parser.print_help()
         sys.exit()
 
+    libraries = args.libraries
+    if operation == 'target' and not libraries:
+        print('No target libraries specified!\n')
+        parser.print_help()
+        sys.exit()
+
     config = args.config
     if not os.path.exists(config):
         print('Config file:', config, 'does not exist')
         sys.exit()
 
-    libraries = args.libraries
-    if operation == 'target' and not libraries:
-        print('No target libraries specified!\n')
-        parser.print_help()
+    # Read config file
+    config_parser = ConfigParser.RawConfigParser()
+    config_parser.optionxform = str
+    sections = ['common', target]
+    try:
+        config_parser.read(config)
+        for section in sections:
+            for option, value in config_parser.items(section):
+                if option == 'SAUCELABS_HUB_URL':
+                    saucelabs_user = os.environ['SAUCELABS_USER']
+                    saucelabs_key = os.environ['SAUCELABS_KEY']
+                    value = value.replace('SAUCELABS_USER', saucelabs_user)
+                    value = value.replace('SAUCELABS_KEY', saucelabs_key)
+                os.environ[option] = value
+    except:
+        print('Error parsing config file:', config)
+        print('Please check the config.cfg.template for the required format')
         sys.exit()
 
-    # Read environment variables file
-    output = subprocess.check_output('source {}; env'.format(config), shell=True, executable=SHELL)
-    env_vars = dict((line.split('=', 1) for line in output.splitlines()))
-    os.environ.update(env_vars)
+    os.environ['CODEBENDER_SELENIUM_HUB_URL'] = os.environ['LOCAL_HUB_URL']
+    if args.saucelabs:
+        os.environ['CODEBENDER_SELENIUM_HUB_URL'] = os.environ['SAUCELABS_HUB_URL']
+
+    capabilities = args.capabilities
+    if capabilities:
+        os.environ['CAPABILITIES'] = capabilities
+
+    # Generate User agents
+    file_path = os.path.join(os.path.dirname(__file__), '..', 'codebender_testing', capabilities)
+    generate_user_agents(file_path)
 
     # Run tests
     tests = Tests(target, config)

codebender_testing/capabilities.yaml

@@ -7,12 +7,10 @@
 #   https://docs.saucelabs.com/reference/test-configuration
 
 - browserName: "firefox"
-  version: 42
+  version: 43
   public: "public restricted"
-  seleniumVersion: "2.48.0"
   maxDuration: 10800
 - browserName: "chrome"
   version: 47
   public: "public restricted"
-  seleniumVersion: "2.50.1"
   maxDuration: 10800

codebender_testing/capabilities_chrome.yaml

@@ -0,0 +1,12 @@
+# This file contains a list of capabilities which will be used to instantiate
+# the remote selenium webdrivers.
+# Each list entry will cause the entire test suite to be run for a remote
+# webdriver with the capabilities specified in the entry.
+
+# See here for more on test configuration:
+#   https://docs.saucelabs.com/reference/test-configuration
+
+- browserName: "chrome"
+  version: 47
+  public: "public restricted"
+  maxDuration: 10800

codebender_testing/capabilities_firefox.yaml

@@ -7,7 +7,6 @@
 #   https://docs.saucelabs.com/reference/test-configuration
 
 - browserName: "firefox"
-  version: 42
+  version: 43
   public: "public restricted"
-  seleniumVersion: "2.48.0"
   maxDuration: 10800

codebender_testing/config.py

@@ -6,7 +6,8 @@
 import yaml
 import simplejson
 import pytest
-
+import string
+import random
 
 def _rel_path(*args):
     """Returns a path relative to config.py file's directory."""
@@ -65,7 +66,7 @@ def jsondump(data):
 CHROME_EXT_MAX_CHROME_VERSION = 41
 
 # Path to YAML file specifying capability list.
-DEFAULT_CAPABILITIES_FILE = os.getenv('CAPABILITIES', 'capabilities.yaml')
+DEFAULT_CAPABILITIES_FILE = os.getenv('CAPABILITIES', 'capabilities_firefox.yaml')
 DEFAULT_CAPABILITIES_FILE_PATH = _rel_path(DEFAULT_CAPABILITIES_FILE)
 
 # Files used for testing.
@@ -79,16 +80,17 @@ def jsondump(data):
 TEST_PROJECT_NAME = "test_project"
 
 TIMEOUT = {
-    'LOCATE_ELEMENT': 30
+    'LOCATE_ELEMENT': 30,
+    'FLASH_FAIL': 30
 }
 
-DEFAULT_USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0 codebender-selenium'
-TESTS_USER_AGENT = os.getenv('SELENIUM_USER_AGENT', DEFAULT_USER_AGENT)
+DEFAULT_USER_AGENT_FIREFOX = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0 codebender-selenium'
+TESTS_USER_AGENT_FIREFOX = os.getenv('SELENIUM_USER_AGENT', DEFAULT_USER_AGENT_FIREFOX)
 
 DEFAULT_USER_AGENT_CHROME = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'
 TESTS_USER_AGENT_CHROME = os.getenv('SELENIUM_USER_AGENT_CHROME', DEFAULT_USER_AGENT_CHROME)
 
-BROWSER = "firefox"
+SELENIUM_BROWSER = 'firefox'
 
 # Set up Selenium Webdrivers to be used for selenium tests.
 def _get_firefox_profile():
@@ -103,6 +105,12 @@ def _get_firefox_profile():
         )
     return firefox_profile
 
+def _get_chrome_profile():
+    """Returns the Chrome profile directory to be used for the Chrome webdriver."""
+    seed = string.ascii_letters + string.digits
+    profile_hash = ''.join(random.choice(seed) for x in range(6))
+    return '.org.chromium.Chromium.' + profile_hash
+
 def get_browsers(capabilities_file_path=None):
     """Returns a list of capabilities. Each item in the list will cause
     the entire suite of tests to be re-run for a browser with those
@@ -131,34 +139,40 @@ def create_webdriver(command_executor, desired_capabilities):
     # Fill in defaults from DesiredCapabilities.{CHROME,FIREFOX} if they are
     # missing from the desired_capabilities dict above.
     _capabilities = desired_capabilities
+    platform = os.getenv('SELENIUM_PLATFORM', None)
+    if platform:
+        _capabilities['platform'] = platform
     browser_profile = None
     browser_profile_path = None
 
+    SELENIUM_BROWSER = browser_name
+
     if browser_name == "chrome":
-        BROWSER = "chrome"
         desired_capabilities = DesiredCapabilities.CHROME.copy()
-        desired_capabilities.update(_capabilities)
+        options = chrome.options.Options()
+        browser_profile_path = os.path.join('/tmp', _get_chrome_profile())
+        options.add_argument('--user-data-dir=' + browser_profile_path)
         if desired_capabilities["version"] > CHROME_EXT_MAX_CHROME_VERSION:
-            # Add new chrome extension to capabilities.
-            options = chrome.options.Options()
+            # Add the chrome app to capabilities.
             options.add_extension(os.path.join(_EXTENSIONS_DIR, _CHROME_APP_FNAME))
             options.add_argument("--user-agent=" + TESTS_USER_AGENT_CHROME)
-            desired_capabilities.update(options.to_capabilities())
-            desired_capabilities.update(_capabilities)
         else:
             raise ValueError("The testing suite only supports Chrome versions greater than v%d, "
                             "but v%d was specified. Please specify a higher version number."
                             % (CHROME_EXT_MAX_CHROME_VERSION, desired_capabilities["version"]))
+        desired_capabilities.update(options.to_capabilities())
+        desired_capabilities.update(_capabilities)
 
     elif browser_name == "firefox":
         desired_capabilities = DesiredCapabilities.FIREFOX.copy()
         desired_capabilities.update(_capabilities)
         browser_profile = _get_firefox_profile()
         browser_profile_path = browser_profile.path
-        browser_profile.set_preference("general.useragent.override", TESTS_USER_AGENT)
+        browser_profile.set_preference("general.useragent.override", TESTS_USER_AGENT_FIREFOX)
         desired_capabilities["firefox_profile"] = browser_profile.update_preferences()
     else:
         raise ValueError("Invalid webdriver %s (only chrome and firefox are supported)" % browser_name)
+
     return {
         'driver': webdriver.Remote(
             command_executor=command_executor,