Merge remote-tracking branch 'origin/bduffany-compile-tests' into development

Author: freskoulix

codebender_testing/config.py

@@ -6,6 +6,21 @@
 # URL of the site to be used for testing
 BASE_URL = "http://localhost"
 
+# User whose projects we'd like to compile in our compile_tester
+# test case(s).
+COMPILE_TESTER_URL = "/user/cb_compile_tester"
+
+# The prefix for all filenames of log files.
+# Note that it is given as a time format string, which will
+# be formatted appropriately.
+LOGFILE_PREFIX = os.path.join("logs", "%Y-%m-%d_%H-%M-%S-{log_name}.json")
+
+# Logfile for COMPILE_TESTER compilation results
+COMPILE_TESTER_LOGFILE = LOGFILE_PREFIX.format(log_name="cb_compile_tester")
+
+# Logfile for /libraries compilation results
+LIBRARIES_TEST_LOGFILE = LOGFILE_PREFIX.format(log_name="libraries_test")
+
 # URL of the actual Codebender website
 LIVE_SITE_URL = "http://codebender.cc"
 

codebender_testing/utils.py

@@ -1,7 +1,12 @@
+from time import gmtime
+from time import strftime
+import json
 import re
 
 from selenium import webdriver
 from selenium.common.exceptions import NoSuchElementException
+from selenium.common.exceptions import StaleElementReferenceException
+from selenium.common.exceptions import WebDriverException
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
 from selenium.webdriver.support import expected_conditions
@@ -15,18 +20,51 @@
 from codebender_testing.config import WEBDRIVERS
 
 
+# Time to wait until we give up on a DOM property becoming available.
+DOM_PROPERTY_DEFINED_TIMEOUT = 10
+
+# JavaScript snippet to extract all the links to sketches on the current page.
+# `selector` is a CSS selector selecting these links.
+_GET_SKETCHES_SCRIPT = \
+    "return $('{selector}').map(function() {{ return this.href; }}).toArray();"
+
+# JavaScript snippet to verify the code on the current page.
+_VERIFY_SCRIPT = """
+if (window.compilerflasher !== undefined) {
+    compilerflasher.verify();
+} else {
+    // BACHELOR
+    verify();
+}
+"""
+
+# How long (in seconds) to wait before assuming that an example
+# has failed to compile
+VERIFY_TIMEOUT = 15
+
+# Messages displayed to the user after verifying a sketch.
+VERIFICATION_SUCCESSFUL_MESSAGE = "Verification Successful"
+VERIFICATION_FAILED_MESSAGE = "Verification failed."
+
+
 class SeleniumTestCase(object):
     """Base class for all Selenium tests."""
 
+    # This can be configured on a per-test case basis to use a different
+    # URL for testing; e.g., http://localhost, or http://codebender.cc.
+    # It is set via command line option in _testcase_attrs (below)
+    site_url = None
+
     @classmethod
     @pytest.fixture(scope="class", autouse=True)
-    def _testcase_attrs(cls, webdriver):
+    def _testcase_attrs(cls, webdriver, testing_url):
         """Sets up any class attributes to be used by any SeleniumTestCase.
         Here, we just store fixtures as class attributes. This allows us to avoid
         the pytest boilerplate of getting a fixture value, and instead just
         refer to the fixture as `self.<fixture>`.
         """
         cls.driver = webdriver
+        cls.site_url = testing_url
 
     @pytest.fixture(scope="class")
     def tester_login(self):
@@ -40,12 +78,12 @@ def open(self, url=None):
         """
         if url is None:
             url = ''
-        if re.match(".+?://^", url):
+        if re.match(".+?://", url):
             # url specifies an absolute path.
             return self.driver.get(url)
         else:
             url = url.lstrip('/')
-            return self.driver.get("%s/%s" % (BASE_URL, url))
+            return self.driver.get("%s/%s" % (self.site_url, url))
 
     def open_project(self, project_name=None):
         """Opens the project specified by `name`, bringing the driver to the
@@ -91,3 +129,107 @@ def delete_project(self, project_name):
         delete_button.click()
         popup_delete_button = self.get_element(By.ID, 'deleteProjectButton')
         popup_delete_button.click()
+
+    def compile_sketch(self, url, iframe=False):
+        """Compiles the sketch located at `url`, or an iframe within the page
+        referred to by `url`.  Raises an exception if it does not compile.
+        """
+        self.open(url)
+        if iframe:
+            # Note: here, we simply get the first iframe on the page.
+            # There's a slightly less awkward way to do this that involves
+            # giving this iframe a meaningful name in the HTML (TODO?)
+            self.driver.switch_to_frame(0)
+        self.execute_script(_VERIFY_SCRIPT)
+        # In the BACHELOR site the id is 'operation_output', but in the live
+        # site the id is 'cb_cf_operation_output'. The [id$=operation_output]
+        # here selects an id that _ends_ with 'operation_output'.
+        compile_result = WebDriverWait(self.driver, VERIFY_TIMEOUT).until(
+            any_text_to_be_present_in_element((By.CSS_SELECTOR, "[id$=operation_output]"),
+                VERIFICATION_SUCCESSFUL_MESSAGE, VERIFICATION_FAILED_MESSAGE))
+        if compile_result != VERIFICATION_SUCCESSFUL_MESSAGE:
+            raise VerificationError(compile_result)
+
+
+    def compile_all_sketches(self, url, selector, iframe=False, logfile=None):
+        """Compiles all projects on the page at `url`. `selector` is a CSS selector
+        that should select all relevant <a> tags containing links to sketches.
+        `logfile` specifies a path to a file to which test results will be
+        logged. If it is not `None`, compile errors will not cause the test
+        to halt, but rather be logged to the given file. `logfile` may be a time
+        format string, which will be formatted appropriately.
+        `iframe` specifies whether the urls pointed to by `selector` are contained
+        within an iframe.
+        """
+        self.open(url)
+        sketches = self.execute_script(_GET_SKETCHES_SCRIPT.format(selector=selector))
+        assert len(sketches) > 0
+
+        if logfile is None:
+            for sketch in sketches:
+                self.compile_sketch(sketch, iframe=iframe)
+        else:
+            log_entry = {'url': self.site_url, 'succeeded': [], 'failed': []}
+            for sketch in sketches:
+                try:
+                    self.compile_sketch(sketch, iframe=iframe)
+                    log_entry['succeeded'].append(sketch)
+                except (VerificationError, WebDriverException) as e:
+                    log_entry['failed'].append({
+                        'sketch': sketch,
+                        'exception': "%s; %s" % (type(e).__name__, str(e))
+                        # TODO?: is it possible to get the actual compiler error?
+                    })
+            # Dump the test results to `logfile`.
+            f = open(strftime(logfile, gmtime()), 'w')
+            json.dump(log_entry, f)
+            f.close()
+
+
+    def execute_script(self, script, *deps):
+        """Waits for all JavaScript variables in `deps` to be defined, then
+        executes the given script. Especially useful for waiting for things like
+        jQuery to become available for use."""
+        if len(deps) > 0:
+            WebDriverWait(self.driver, DOM_PROPERTY_DEFINED_TIMEOUT).until(
+                dom_properties_defined(*deps))
+        return self.driver.execute_script(script)
+
+
+class VerificationError(Exception):
+    """An exception representing a failed verification of a sketch."""
+    pass
+
+
+class dom_properties_defined(object):
+    """An expectation for the given DOM properties to be defined.
+    See selenium.webdriver.support.expected_conditions for more on how this
+    type of class works.
+    """
+
+    def __init__(self, *properties):
+        self._properties = properties
+
+    def __call__(self, driver):
+        return all(
+            driver.execute_script("return window.%s !== undefined" % prop)
+            for prop in self._properties)
+
+
+class any_text_to_be_present_in_element(object):
+    """An expectation for checking if any of the given strings are present in
+    the specified element. Returns the string that was present.
+    """
+    def __init__(self, locator, *texts):
+        self.locator = locator
+        self.texts = texts
+
+    def __call__(self, driver):
+        try :
+            element_text = expected_conditions._find_element(driver, self.locator).text
+            for text in self.texts:
+                if text in element_text:
+                    return text
+            return False
+        except StaleElementReferenceException:
+            return False

logs/.gitignore

@@ -0,0 +1,8 @@
+# Ignore everything in this directory
+*
+
+# Except this file
+!.gitignore
+
+# We need this gitignore since the logs directory is required for storing logs,
+# but we don't want to commit the logs themselves.

tests/compile_tester/__init__.py

@@ -0,0 +1,18 @@
+import pytest
+
+from codebender_testing.config import COMPILE_TESTER_LOGFILE
+from codebender_testing.config import COMPILE_TESTER_URL
+from codebender_testing.config import LIVE_SITE_URL
+from codebender_testing.utils import SeleniumTestCase
+
+
+class TestCompileTester(SeleniumTestCase):
+
+    # Here, we require the LIVE_SITE_URL since the compiler tester user
+    # does not exist in the bachelor version.
+    @pytest.mark.requires_url(LIVE_SITE_URL)
+    def test_compile_all_user_projects(self):
+        """Tests that all library examples compile successfully."""
+        self.compile_all_sketches(COMPILE_TESTER_URL, '#user_projects tbody a',
+            iframe=True, log_file=COMPILE_TESTER_LOGFILE)
+

tests/compile_tester/test_compile_tester_projects.py

@@ -1,5 +1,6 @@
 import pytest
 
+from codebender_testing.config import BASE_URL
 from codebender_testing.config import WEBDRIVERS
 
 
@@ -13,3 +14,20 @@ def webdriver(request):
     request.addfinalizer(lambda: driver.quit())
     return driver
 
+def pytest_addoption(parser):
+    """Adds command line options to py.test."""
+    parser.addoption("--url", action="store", default=BASE_URL,
+                     help="URL to use for testing, e.g. http://localhost, http://codebender.cc")
+
+@pytest.fixture(scope="class")
+def testing_url(request):
+    return request.config.getoption("--url")
+
+@pytest.fixture(autouse=True)
+def skip_by_site(request, testing_url):
+    """Skips tests that require a certain site URL in order to run properly."""
+    if request.node.get_marker('requires_url'):
+        required_url = request.node.get_marker('requires_url').args[0]
+        if required_url.rstrip('/') != testing_url.rstrip('/'):
+            pytest.skip('skipped test that requires --url=%s')
+

tests/conftest.py

@@ -0,0 +1,15 @@
+from codebender_testing.config import LIBRARIES_TEST_LOGFILE
+from codebender_testing.config import LIVE_SITE_URL
+from codebender_testing.utils import SeleniumTestCase
+
+
+class TestLibraryExamples(SeleniumTestCase):
+
+    # Overrides the site_url in SeleniumTestCase.
+    site_url = LIVE_SITE_URL
+
+    def test_compile_all_libraries(self):
+        """Tests that all library examples compile successfully."""
+        self.compile_all_sketches('/libraries', '.accordion li a',
+                                  logfile=LIBRARIES_TEST_LOGFILE)
+