Improve the logging system

Author: mdmintz

seleniumbase/core/log_helper.py

@@ -105,6 +105,47 @@ def get_html_source_with_base_href(driver, page_source):
     return ''
 
 
+def copytree(src, dst, symlinks=False, ignore=None):
+    if not os.path.exists(dst):
+        os.makedirs(dst)
+    for item in os.listdir(src):
+        s = os.path.join(src, item)
+        d = os.path.join(dst, item)
+        if os.path.isdir(s):
+            copytree(s, d, symlinks, ignore)
+        else:
+            if not os.path.exists(d) or (
+                    os.stat(s).st_mtime - os.stat(d).st_mtime > 1):
+                shutil.copy2(s, d)
+
+
+def archive_logs_if_set(log_path, archive_logs=False):
+    """ Handle Logging """
+    if "-n" in sys.argv or "".join(sys.argv) == "-c":
+        return  # Skip if multithreaded
+    if log_path.endswith("/"):
+        log_path = log_path[:-1]
+    if not os.path.exists(log_path):
+        try:
+            os.makedirs(log_path)
+        except Exception:
+            pass  # Only reachable during multi-threaded runs
+    else:
+        if settings.ARCHIVE_EXISTING_LOGS or archive_logs:
+            if len(os.listdir(log_path)) > 0:
+                archived_folder = "%s/../archived_logs/" % log_path
+                archived_folder = os.path.realpath(archived_folder) + '/'
+                log_path = os.path.realpath(log_path) + '/'
+                if not os.path.exists(archived_folder):
+                    try:
+                        os.makedirs(archived_folder)
+                    except Exception:
+                        pass  # Only reachable during multi-threaded runs
+                time_id = str(int(time.time()))
+                archived_logs = "%slogs_%s" % (archived_folder, time_id)
+                copytree(log_path, archived_logs)
+
+
 def log_folder_setup(log_path, archive_logs=False):
     """ Handle Logging """
     if log_path.endswith("/"):
@@ -116,30 +157,23 @@ def log_folder_setup(log_path, archive_logs=False):
             pass  # Should only be reachable during multi-threaded runs
     else:
         archived_folder = "%s/../archived_logs/" % log_path
+        archived_folder = os.path.realpath(archived_folder) + '/'
         if not os.path.exists(archived_folder):
             try:
                 os.makedirs(archived_folder)
             except Exception:
                 pass  # Should only be reachable during multi-threaded runs
-        if not "".join(sys.argv) == "-c":
-            # Only move log files if the test run is not multi-threaded.
-            # (Running tests with "-n NUM" will create threads that only
-            # have "-c" in the sys.argv list. Easy to catch.)
-            archived_logs = "%slogs_%s" % (
-                archived_folder, int(time.time()))
-            if "_logs" not in log_path:
-                # Don't move files in a custom-named log folder (in case
-                # the user specifed a folder with important files in it)
-                # unless the folder name contains "_logs".
-                # The default name for the log folder is "latest_logs".
-                return
+        archived_logs = "%slogs_%s" % (
+            archived_folder, int(time.time()))
+
+        if len(os.listdir(log_path)) > 0:
             shutil.move(log_path, archived_logs)
             os.makedirs(log_path)
             if not settings.ARCHIVE_EXISTING_LOGS and not archive_logs:
                 shutil.rmtree(archived_logs)
-            elif len(os.listdir(archived_logs)) == 0:
-                # Don't archive an empty directory
-                shutil.rmtree(archived_logs)
             else:
-                # Logs are saved/archived
-                pass
+                if ("-n" in sys.argv or "".join(sys.argv) == "-c"):
+                    # Logs are saved/archived now if tests are multithreaded
+                    pass
+                else:
+                    shutil.rmtree(archived_logs)  # (Archive test run later)

seleniumbase/plugins/base_plugin.py

@@ -1,7 +1,6 @@
 # -*- coding: utf-8 -*-
 """ This is the Nose plugin for setting a test environment and saving logs. """
 
-import os
 import sys
 import time
 from nose.plugins import Plugin
@@ -17,7 +16,6 @@ class Base(Plugin):
     --env=ENV  (Set a test environment. Use "self.env" to use this in tests.)
     --data=DATA  (Extra data to pass to tests. Use "self.data" in tests.)
     --settings-file=FILE  (Overrides SeleniumBase settings.py values.)
-    --log-path=LOG_PATH  (The directory where log files get saved to.)
     --archive-logs  (Archive old log files instead of deleting them.)
     --report  (The option to create a fancy report after tests complete.)
     --show-report   If self.report is turned on, then the report will
@@ -59,7 +57,7 @@ def options(self, parser, env):
             '--log_path', '--log-path',
             dest='log_path',
             default='latest_logs/',
-            help='Where the log files are saved.')
+            help='Where the log files are saved. (No longer editable!)')
         parser.add_option(
             '--archive_logs', '--archive-logs',
             action="store_true",
@@ -102,16 +100,13 @@ def configure(self, options, conf):
         self.page_results_list = []
         self.test_count = 0
         self.import_error = False
-        log_path = options.log_path
+        log_path = 'latest_logs/'
         archive_logs = options.archive_logs
         log_helper.log_folder_setup(log_path, archive_logs)
         if self.report_on:
             report_helper.clear_out_old_report_logs(archive_past_runs=False)
 
     def beforeTest(self, test):
-        test_logpath = self.options.log_path + "/" + test.id()
-        if not os.path.exists(test_logpath):
-            os.makedirs(test_logpath)
         test.test.environment = self.options.environment
         test.test.env = self.options.environment  # Add a shortened version
         test.test.data = self.options.data
@@ -123,6 +118,8 @@ def beforeTest(self, test):
         self.start_time = float(time.time())
 
     def finalize(self, result):
+        log_helper.archive_logs_if_set(
+            self.options.log_path, self.options.archive_logs)
         if self.report_on:
             if not self.import_error:
                 report_helper.add_bad_page_log_file(self.page_results_list)

seleniumbase/plugins/pytest_plugin.py

@@ -29,7 +29,6 @@ def pytest_addoption(parser):
     --headless  (The option to run tests headlessly. The default on Linux OS.)
     --headed  (The option to run tests with a GUI on Linux OS.)
     --start-page=URL  (The starting URL for the web browser when tests begin.)
-    --log-path=LOG_PATH  (The directory where log files get saved to.)
     --archive-logs  (Archive old log files instead of deleting them.)
     --slow  (The option to slow down the automation.)
     --demo  (The option to visually see test actions as they occur.)
@@ -113,7 +112,7 @@ def pytest_addoption(parser):
     parser.addoption('--log_path', '--log-path',
                      dest='log_path',
                      default='latest_logs/',
-                     help='Where the log files are saved.')
+                     help='Where log files are saved. (No longer editable!)')
     parser.addoption('--archive_logs', '--archive-logs',
                      action="store_true",
                      dest='archive_logs',
@@ -380,7 +379,7 @@ def pytest_configure(config):
     sb_config.settings_file = config.getoption('settings_file')
     sb_config.user_data_dir = config.getoption('user_data_dir')
     sb_config.database_env = config.getoption('database_env')
-    sb_config.log_path = config.getoption('log_path')
+    sb_config.log_path = 'latest_logs/'  # (No longer editable!)
     sb_config.archive_logs = config.getoption('archive_logs')
     sb_config.slow_mode = config.getoption('slow_mode')
     sb_config.demo_mode = config.getoption('demo_mode')
@@ -433,6 +432,7 @@ def pytest_unconfigure():
             except Exception:
                 pass
         sb_config.shared_driver = None
+    log_helper.archive_logs_if_set(sb_config.log_path, sb_config.archive_logs)
 
 
 def pytest_runtest_setup():