Merge pull request #21 from codebendercc/tests_additions_and_fixes

Various additions and fixes.

Author: dastergon

bin/seleniumbender

@@ -64,7 +64,6 @@ done
 cd ..
 
 URL="https://codebender.cc"
-SOURCE="codebender_cc"
 
 email_date=$(date +"%Y-%m-%d %H:%M:%S")
 
@@ -73,40 +72,43 @@ NO_LOGS=0 # flag for tests that do not send any logs on their email
 
 if [ "${target}" -eq 1 ]; then
     IDENTIFIER="test_target_libraries"
-    tox tests/target_libraries -- --url=${URL} --source=${SOURCE} -F --plugin --libraries=${TARGETS}
+    tox tests/target_libraries -- --url=${URL} -F --plugin --libraries=${TARGETS}
 elif [ "${examples}" -eq 1 ]; then
     IDENTIFIER="libraries_test"
-    tox tests/libraries -- --url=${URL} --source=${SOURCE} -F --plugin
+    tox tests/libraries -- --url=${URL} -F --plugin
 elif [ "${libraries}" -eq 1 ]; then
     IDENTIFIER="libraries_fetch"
-    tox tests/libraries_fetch -- --url=${URL} --source=${SOURCE} -F --plugin
+    tox tests/libraries_fetch -- --url=${URL} -F --plugin
 elif [ "${sketches}" -eq 1 ]; then
     IDENTIFIER="cb_compile_tester"
-    tox tests/compile_tester -- --url=${URL} --source=${SOURCE} -F --plugin
+    tox tests/compile_tester -- --url=${URL} -F --plugin
 elif [ "${common}" -eq 1 ]; then
     IDENTIFIER="common"
-    tox tests/common -- --url=${URL} --source=${SOURCE} --plugin
+    tox tests/common -- --url=${URL} --plugin
     RETVAL=$?
     NO_LOGS=1
 elif [ "${noplugin}" -eq 1 ]; then
     IDENTIFIER="noplugin"
-    tox tests/noplugin -- --url=${URL} --source=${SOURCE}
+    tox tests/noplugin -- --url=${URL}
     RETVAL=$?
     NO_LOGS=1
 elif [ "${walkthrough}" -eq 1 ]; then
     IDENTIFIER="walkthrough"
     RETVALS=()
     # Linux
+    export SELENIUM_USER_AGENT_CHROME='Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36 codebender-selenium'
     export SELENIUM_USER_AGENT='Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0 codebender-selenium'
-    tox tests/walkthrough -- --url=${URL} --source=${SOURCE} --plugin
+    tox tests/walkthrough -- --url=${URL} --plugin
     RETVALS+=($?)
     # Windows
+    export SELENIUM_USER_AGENT_CHROME='Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36 codebender-selenium'
     export SELENIUM_USER_AGENT='Mozilla/5.0 (Windows NT 6.1; rv:43.0) Gecko/20100101 Firefox/43.0 codebender-selenium'
-    tox tests/walkthrough -- --url=${URL} --source=${SOURCE} --plugin
+    tox tests/walkthrough -- --url=${URL} --plugin
     RETVALS+=($?)
     # MacOSX
+    export SELENIUM_USER_AGENT_CHROME='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36 codebender-selenium'
     export SELENIUM_USER_AGENT='Mozilla/5.0 (Macintosh; Intel Mac OS X 10_11_1; rv:43.0) Gecko/20100101 Firefox/43.0 codebender-selenium'
-    tox tests/walkthrough -- --url=${URL} --source=${SOURCE} --plugin
+    tox tests/walkthrough -- --url=${URL} --plugin
     RETVALS+=($?)
 
     for i in "${RETVALS[@]}"
@@ -119,7 +121,7 @@ elif [ "${walkthrough}" -eq 1 ]; then
 elif [ "${staging}" -eq 1 ]; then
     IDENTIFIER="cb_compile_tester_staging"
     URL="https://staging.codebender.cc"
-    tox tests/compile_tester -- --url=${URL} --source=${SOURCE} -F --plugin
+    tox tests/compile_tester -- --url=${URL} -F --plugin
     # No need to send email for tests in staging
     exit $?
 fi

codebender_testing/capabilities.yaml

@@ -12,7 +12,7 @@
   seleniumVersion: "2.48.0"
   maxDuration: 10800
 - browserName: "chrome"
-  version: 41
+  version: 47
   public: "public restricted"
-  seleniumVersion: "2.48.0"
+  seleniumVersion: "2.50.1"
   maxDuration: 10800

codebender_testing/config.py

@@ -9,10 +9,13 @@
 
 
 def _rel_path(*args):
-    """Forms a path relative to this file's directory."""
+    """Returns a path relative to config.py file's directory."""
     return os.path.join(os.path.dirname(__file__), *args)
 
 def get_path(directory, filename=None):
+    """Returns an absolute path as seen from your current working directory.
+    If a file is passed as argument, returns an aboslute path including the
+    file."""
     path = os.path.join(os.path.dirname( __file__ ), '..', directory)
     if filename:
         path = os.path.join(os.path.dirname( __file__ ), '..', directory, filename)
@@ -21,16 +24,12 @@ def get_path(directory, filename=None):
 def jsondump(data):
     return simplejson.dumps(data, sort_keys=True, indent=4 * ' ')
 
-# URL of the default site to be used for testing
+# URL of the default site to be used for testing.
 BASE_URL = "http://localhost"
-# URL of the actual Codebender website
+# URL of the actual Codebender website.
 LIVE_SITE_URL = "https://codebender.cc"
 STAGING_SITE_URL = "https://staging.codebender.cc"
 
-# Names of sources (i.e. repositories) used to generate the codebender site.
-SOURCE_BACHELOR = 'bachelor'
-SOURCE_CODEBENDER_CC = 'codebender_cc'
-
 # User whose projects we'd like to compile in our compile_tester
 # test case(s).
 COMPILE_TESTER_URL = "/user/cb_compile_tester"
@@ -41,42 +40,42 @@ def jsondump(data):
 # be formatted appropriately.
 LOGFILE_PREFIX = _rel_path("..", "logs", "%Y-%m-%d_%H-%M-%S-{log_name}.json")
 
-# Logfile for COMPILE_TESTER compilation results
+# Logfile for COMPILE_TESTER compilation results.
 COMPILE_TESTER_LOGFILE = LOGFILE_PREFIX.format(log_name="cb_compile_tester")
 COMPILE_TESTER_LOGFILE_STAGING = LOGFILE_PREFIX.format(log_name="staging_cb_compile_tester")
 
-# Logfile for /libraries compilation results
+# Logfile for /libraries compilation results.
 LIBRARIES_TEST_LOGFILE = LOGFILE_PREFIX.format(log_name="libraries_test")
 
-# Logfile for /libraries fetch results
+# Logfile for /libraries fetch results.
 LIBRARIES_FETCH_LOGFILE = LOGFILE_PREFIX.format(log_name="libraries_fetch")
 
+# Directory in which Firefox and Chrome extensions are stored.
 _EXTENSIONS_DIR = _rel_path('..', 'extensions')
+
+#  Firefox plugin for all Firefox versions.
 _FIREFOX_EXTENSION_FNAME = 'codebender.xpi'
+# Chrome extension for Chrome versions < 42.
 _CHROME_EXTENSION_FNAME = 'codebendercc-extension.crx'
+# Chrome extension for Chrome versions >= 42.
+_CHROME_APP_FNAME = 'chrome-app-1.0.0.8.zip'
 
 # Maximum version number that we can use the Chrome extension with.
-# For versions higher than this, we need to use the newer Codebender app
+# For versions higher than this, we need to use the newer Codebender app.
 CHROME_EXT_MAX_CHROME_VERSION = 41
 
-# Path to YAML file specifying capability list
+# Path to YAML file specifying capability list.
 DEFAULT_CAPABILITIES_FILE = os.getenv('CAPABILITIES', 'capabilities.yaml')
 DEFAULT_CAPABILITIES_FILE_PATH = _rel_path(DEFAULT_CAPABILITIES_FILE)
 
-# Files used for testing
+# Files used for testing.
 TEST_DATA_DIR = _rel_path('..', 'test_data')
 TEST_DATA_INO = os.path.join(TEST_DATA_DIR, 'upload_ino.ino')
 TEST_DATA_ZIP = os.path.join(TEST_DATA_DIR, 'upload_zip.zip')
 
 # Directory in which the local compile tester files are stored.
 COMPILE_TESTER_DIR = os.path.join(TEST_DATA_DIR, 'cb_compile_tester')
 
-# Credentials to use when logging into the bachelor site
-TEST_CREDENTIALS = {
-    "username": "tester",
-    "password": "testerPASS"
-}
-
 TEST_PROJECT_NAME = "test_project"
 
 TIMEOUT = {
@@ -86,7 +85,12 @@ def jsondump(data):
 DEFAULT_USER_AGENT = 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0 codebender-selenium'
 TESTS_USER_AGENT = os.getenv('SELENIUM_USER_AGENT', DEFAULT_USER_AGENT)
 
-# Set up Selenium Webdrivers to be used for selenium tests
+DEFAULT_USER_AGENT_CHROME = 'Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/47.0.2526.106 Safari/537.36'
+TESTS_USER_AGENT_CHROME = os.getenv('SELENIUM_USER_AGENT_CHROME', DEFAULT_USER_AGENT_CHROME)
+
+BROWSER = "firefox"
+
+# Set up Selenium Webdrivers to be used for selenium tests.
 def _get_firefox_profile():
     """Returns the Firefox profile to be used for the FF webdriver.
     Specifically, we're equipping the webdriver with the Codebender
@@ -131,26 +135,20 @@ def create_webdriver(command_executor, desired_capabilities):
     browser_profile_path = None
 
     if browser_name == "chrome":
+        BROWSER = "chrome"
         desired_capabilities = DesiredCapabilities.CHROME.copy()
         desired_capabilities.update(_capabilities)
-
-        # NOTE: the following logic is disabled since the remote webdriver is
-        # not properly installing the codebender extension. It is kept for
-        # reference until we can figure out how to properly add the Chrome
-        # extension.
-
-        # # Add chrome extension to capabilities
-        # options = chrome.options.Options()
-        # options.add_extension(os.path.join(_EXTENSIONS_DIR, _CHROME_EXTENSION_FNAME))
-        # desired_capabilities.update(options.to_capabilities())
-        # # Right now we only support up to v41 for this testing suite.
-        # if "version" in desired_capabilities:
-        #     if desired_capabilities["version"] > CHROME_EXT_MAX_CHROME_VERSION:
-        #         raise ValueError("The testing suite only supports Chrome versions up to v%d, "
-        #                          "but v%d was specified. Please specify a lower version "
-        #                          "number." % (CHROME_EXT_MAX_CHROME_VERSION, desired_capabilities["version"]))
-        # else:
-        #     desired_capabilities["version"] = CHROME_EXT_MAX_CHROME_VERSION
+        if desired_capabilities["version"] > CHROME_EXT_MAX_CHROME_VERSION:
+            # Add new chrome extension to capabilities.
+            options = chrome.options.Options()
+            options.add_extension(os.path.join(_EXTENSIONS_DIR, _CHROME_APP_FNAME))
+            options.add_argument("--user-agent=" + TESTS_USER_AGENT_CHROME)
+            desired_capabilities.update(options.to_capabilities())
+            desired_capabilities.update(_capabilities)
+        else:
+            raise ValueError("The testing suite only supports Chrome versions greater than v%d, "
+                            "but v%d was specified. Please specify a higher version number."
+                            % (CHROME_EXT_MAX_CHROME_VERSION, desired_capabilities["version"]))
 
     elif browser_name == "firefox":
         desired_capabilities = DesiredCapabilities.FIREFOX.copy()

codebender_testing/utils.py

@@ -23,7 +23,6 @@
 
 from codebender_testing.config import BASE_URL
 from codebender_testing.config import TIMEOUT
-from codebender_testing.config import TEST_CREDENTIALS
 from codebender_testing.config import TEST_PROJECT_NAME
 from codebender_testing.config import get_path
 from codebender_testing.config import jsondump
@@ -314,8 +313,6 @@ def login(self, credentials=None):
         unspecified location when calling this function.
         `credentials` should be a dict with keys 'username' and 'password',
         mapped to the appropriate values."""
-        if credentials is None:
-            credentials = TEST_CREDENTIALS
         try:
             self.open()
             login_button = self.driver.find_element_by_id('login_btn')
@@ -336,6 +333,7 @@ def logout(self):
         try:
             logout_button = self.driver.find_element_by_id("logout")
             logout_button.send_keys(Keys.ENTER)
+            self.get_element(By.CSS_SELECTOR, '#login_btn')
         except NoSuchElementException:
             # 'Log out' is not displayed, so we're already logged out.
             pass
@@ -496,9 +494,7 @@ def compile_sketch(self, url, boards, iframe=False, project_view=False):
             try:
                 self.execute_script(SELECT_BOARD_SCRIPT(board), '$', 'compilerflasher.pluginHandler.plugin_found')
                 self.execute_script(_VERIFY_SCRIPT, 'compilerflasher')
-                # In the BACHELOR site the id is 'operation_output', but in the live
-                # site the id is 'cb_cf_operation_output'. The [id$=operation_output]
-                # here selects an id that _ends_ with 'operation_output'.
+                # The [id$=operation_output] here selects an id that _ends_ with 'operation_output'.
                 compile_result = WebDriverWait(self.driver, VERIFY_TIMEOUT).until(
                     any_text_to_be_present_in_element(
                         (By.CSS_SELECTOR, "[id$=operation_output]"),

extensions/chrome-app-1.0.0.8.zip

@@ -13,8 +13,8 @@ def test_navigate_home(self, tester_logout):
 
     def test_login(self, tester_logout):
         credentials = {
-            'username': os.environ.get('CODEBENDER_TEST_USER', config.TEST_CREDENTIALS['username']),
-            'password': os.environ.get('CODEBENDER_TEST_PASS', config.TEST_CREDENTIALS['password']),
+            'username': os.environ.get('CODEBENDER_TEST_USER'),
+            'password': os.environ.get('CODEBENDER_TEST_PASS'),
         }
         driver = self.driver
         self.open("/")

extensions/codebender-app.crx

@@ -54,13 +54,11 @@ def test_boards_dropdown(self):
 
         assert boards_dropdown.first_selected_option.text == TEST_BOARD
 
-    @pytest.mark.requires_extension
     def test_ports_dropdown(self):
         """Tests that the ports dropdown exists."""
         ports = self.get_element(By.ID, "cb_cf_ports")
         assert ports.text == 'No ports detected'
 
-    @pytest.mark.requires_extension
     def test_run_with_no_port(self):
         """Makes sure that there is an error when we attempt to run with no
         port selected."""
@@ -72,13 +70,11 @@ def test_run_with_no_port(self):
             )
         )
 
-    @pytest.mark.requires_extension
     def test_speeds_dropdown(self):
         """Tests that the speeds dropdown exists."""
         self.get_element(By.ID, "serial_monitor_toggle").click()
         self.get_element(By.ID, "cb_cf_baud_rates")
 
-    @pytest.mark.requires_extension
     def test_serial_monitor_disables_fields(self):
         """Tests that opening the serial monitor disables the port and baudrate
         fields."""

tests/common/home/test_home.py

@@ -3,8 +3,6 @@
 
 from codebender_testing.config import TEST_DATA_INO
 from codebender_testing.config import TEST_DATA_ZIP
-from codebender_testing.config import SOURCE_BACHELOR
-from codebender_testing.config import SOURCE_CODEBENDER_CC
 from codebender_testing.utils import SeleniumTestCase
 
 
@@ -19,45 +17,6 @@ def open_user_home(self, tester_login):
         performing any of these tests."""
         pass
 
-    @pytest.mark.requires_source(SOURCE_BACHELOR)
-    def test_create_project_blank_name(self):
-        """Test that we get an error when creating a project with no name."""
-        create_button = self.get_element(By.CSS_SELECTOR, '.form-search button')
-        create_button.click()
-        error_heading = self.get_element(By.CSS_SELECTOR, '.alert h4')
-        assert error_heading.text.startswith('Error')
-
-    @pytest.mark.requires_source(SOURCE_BACHELOR)
-    def test_create_project_invalid_name(self):
-        """Test that we get an error when creating a project with an
-        invalid name (e.g., a name containing a backslash).
-        """
-        project_name_input = self.get_element(By.CSS_SELECTOR,
-                                              '.form-search input[type=text]')
-        project_name_input.clear()
-        project_name_input.send_keys('foo\\bar')
-        create_button = self.get_element(By.CSS_SELECTOR, '.form-search button')
-        create_button.click()
-
-        error_heading = self.get_element(By.CSS_SELECTOR, '.alert h4')
-        assert error_heading.text.startswith('Error')
-
-    @pytest.mark.requires_source(SOURCE_BACHELOR)
-    def test_create_project_valid_name(self):
-        """Test that we can successfully create a project with a valid name."""
-        project_name_input = self.get_element(By.CSS_SELECTOR,
-                                              '.form-search input[type=text]')
-        project_name_input.clear()
-        project_name_input.send_keys(NEW_PROJECT_NAME)
-        create_button = self.get_element(By.CSS_SELECTOR, '.form-search button')
-        create_button.click()
-
-        project_heading = self.get_element(By.ID, 'editor_heading_project_name')
-        assert project_heading.text == NEW_PROJECT_NAME
-
-        # Cleanup: delete the project we just created.
-        self.delete_project(NEW_PROJECT_NAME)
-
     def _upload_test(self, dropzone_selector, test_fname, sketch_name=None):
         """Tests that we can successfully upload `test_fname`.
         `project_name` is the expected name of the project; by
@@ -66,16 +25,16 @@ def _upload_test(self, dropzone_selector, test_fname, sketch_name=None):
         """
         try:
             upload_name = self.upload_project(dropzone_selector, test_fname, sketch_name)
-            assert upload_name in self.driver.page_source
+            selector = '#project_list li[data-name="'+ str(upload_name.lower()) +'"]'
+            project_uploaded = self.get_element(By.CSS_SELECTOR, selector).text.split('\n')[0]
+            assert upload_name == project_uploaded
         finally:
             self.delete_project(upload_name)
 
-    @pytest.mark.requires_source(SOURCE_CODEBENDER_CC)
     def test_upload_project_ino(self):
         """Tests that we can upload a .ino file."""
         self._upload_test('#uploadInoModal form', TEST_DATA_INO)
 
-    @pytest.mark.requires_source(SOURCE_CODEBENDER_CC)
     def test_upload_project_zip(self):
         """Tests that we can successfully upload a zipped project."""
         # TODO: how is the project name inferred from the zip file?