Add logging for libraries test

Author: Brandon Duffany

codebender_testing/config.py

@@ -10,10 +10,16 @@
 # test case(s).
 COMPILE_TESTER_URL = "/user/cb_compile_tester"
 
-# Path to the logfile for the COMPILE_TESTER.
+# The prefix for all filenames of log files.
 # Note that it is given as a time format string, which will
 # be formatted appropriately.
-COMPILE_TESTER_LOGFILE = "logs/%Y-%m-%d_%H-%M-%S-cb_compile_tester.json"
+LOGFILE_PREFIX = os.path.join("logs", "%Y-%m-%d_%H-%M-%S-{log_name}.json")
+
+# Logfile for COMPILE_TESTER compilation results
+COMPILE_TESTER_LOGFILE = LOGFILE_PREFIX.format(log_name="cb_compile_tester")
+
+# Logfile for /libraries compilation results
+LIBRARIES_TEST_LOGFILE = LOGFILE_PREFIX.format(log_name="libraries_test")
 
 # URL of the actual Codebender website
 LIVE_SITE_URL = "http://codebender.cc"

codebender_testing/utils.py

@@ -29,7 +29,14 @@
     "return $('{selector}').map(function() {{ return this.href; }}).toArray();"
 
 # JavaScript snippet to verify the code on the current page.
-_VERIFY_SCRIPT = "compilerflasher.verify()"
+_VERIFY_SCRIPT = """
+if (window.compilerflasher !== undefined) {
+    compilerflasher.verify();
+} else {
+    // BACHELOR
+    verify();
+}
+"""
 
 # How long (in seconds) to wait before assuming that an example
 # has failed to compile
@@ -136,19 +143,22 @@ def compile_sketch(self, url, iframe=False):
             # giving this iframe a meaningful name in the HTML (TODO?)
             self.driver.switch_to_frame(0)
         self.execute_script(_VERIFY_SCRIPT)
+        # In the BACHELOR site the id is 'operation_output', but in the live
+        # site the id is 'cb_cf_operation_output'. The [id$=operation_output]
+        # here selects an id that _ends_ with 'operation_output'.
         compile_result = WebDriverWait(self.driver, VERIFY_TIMEOUT).until(
-            any_text_to_be_present_in_element((By.ID, "cb_cf_operation_output"),
+            any_text_to_be_present_in_element((By.CSS_SELECTOR, "[id$=operation_output]"),
                 VERIFICATION_SUCCESSFUL_MESSAGE, VERIFICATION_FAILED_MESSAGE))
         if compile_result != VERIFICATION_SUCCESSFUL_MESSAGE:
             raise VerificationError(compile_result)
 
 
-    def compile_all_sketches(self, url, selector, iframe=False, log_file=None):
+    def compile_all_sketches(self, url, selector, iframe=False, logfile=None):
         """Compiles all projects on the page at `url`. `selector` is a CSS selector
         that should select all relevant <a> tags containing links to sketches.
-        `log_file` specifies a path to a file to which test results will be
+        `logfile` specifies a path to a file to which test results will be
         logged. If it is not `None`, compile errors will not cause the test
-        to halt, but rather be logged to the given file. `log_file` may be a time
+        to halt, but rather be logged to the given file. `logfile` may be a time
         format string, which will be formatted appropriately.
         `iframe` specifies whether the urls pointed to by `selector` are contained
         within an iframe.
@@ -157,11 +167,11 @@ def compile_all_sketches(self, url, selector, iframe=False, log_file=None):
         sketches = self.execute_script(_GET_SKETCHES_SCRIPT.format(selector=selector))
         assert len(sketches) > 0
 
-        if log_file is None:
+        if logfile is None:
             for sketch in sketches:
                 self.compile_sketch(sketch, iframe=iframe)
         else:
-            log_entry = {'succeeded': [], 'failed': []}
+            log_entry = {'url': self.site_url, 'succeeded': [], 'failed': []}
             for sketch in sketches:
                 try:
                     self.compile_sketch(sketch, iframe=iframe)
@@ -172,8 +182,8 @@ def compile_all_sketches(self, url, selector, iframe=False, log_file=None):
                         'exception': "%s; %s" % (type(e).__name__, str(e))
                         # TODO?: is it possible to get the actual compiler error?
                     })
-            # Dump the test results to `log_file`.
-            f = open(strftime(log_file, gmtime()), 'w')
+            # Dump the test results to `logfile`.
+            f = open(strftime(logfile, gmtime()), 'w')
             json.dump(log_entry, f)
             f.close()
 

tests/libraries/test_libraries.py

@@ -1,3 +1,4 @@
+from codebender_testing.config import LIBRARIES_TEST_LOGFILE
 from codebender_testing.config import LIVE_SITE_URL
 from codebender_testing.utils import SeleniumTestCase
 
@@ -9,5 +10,6 @@ class TestLibraryExamples(SeleniumTestCase):
 
     def test_compile_all_libraries(self):
         """Tests that all library examples compile successfully."""
-        self.compile_all_sketches('/libraries', '.accordion li a')
+        self.compile_all_sketches('/libraries', '.accordion li a',
+                                  logfile=LIBRARIES_TEST_LOGFILE)