Merge pull request #754 from seleniumbase/update-dashboards

Improve the SeleniumBase Dashboard functionality

Author: mdmintz

README.md

@@ -457,7 +457,7 @@ pytest test_demo_site.py
 ```
 
 
-<h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> Logging / Results from Failing Tests:</h3>
+<h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> Log files from failed tests:</h3>
 
 Let's try an example of a test that fails:
 
@@ -484,7 +484,11 @@ You'll notice that a logs folder, "latest_logs", was created to hold information
 
 <h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> The SeleniumBase Dashboard:</h3>
 
-The ``--dashboard`` option for pytest generates a SeleniumBase Dashboard located at ``dashboard.html``, which updates automatically as tests run and produce results.
+The ``--dashboard`` option for pytest generates a SeleniumBase Dashboard located at ``dashboard.html``, which updates automatically as tests run and produce results. Example:
+
+```bash
+pytest --dashboard --rs --headless
+```
 
 <img src="https://seleniumbase.io/cdn/img/dashboard_1.png" alt="The SeleniumBase Dashboard" title="The SeleniumBase Dashboard" width="360" />
 
@@ -494,7 +498,7 @@ Additionally, you can host your own SeleniumBase Dashboard Server on a port of y
 python -m http.server 1948
 ```
 
-Now you can navigate to ``http://localhost:1948/dashboard.html`` in order to view the served dashboard from a web browser. (Be sure to run that command in the same directory where you ran your tests.)
+Now you can navigate to ``http://localhost:1948/dashboard.html`` in order to view the dashboard as a web app. This requires two different terminal windows: one for running the server, and another for running the tests, which should be run from the same directory.
 
 Here's a full example of what the SeleniumBase Dashboard may look like:
 
@@ -507,7 +511,7 @@ pytest test_suite.py --dashboard --rs --headless
 --------
 
 <a id="creating_visual_reports"></a>
-<h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> Creating Visual Test Suite Reports:</h3>
+<h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> Creating Visual Test Reports:</h3>
 
 <h4><b>Pytest Reports:</b></h4>
 
@@ -519,6 +523,10 @@ pytest test_suite.py --html=report.html
 
 <img src="https://seleniumbase.io/cdn/img/html_report.png" alt="Example Pytest Report" title="Example Pytest Report" width="520" />
 
+When combining pytest html reports with SeleniumBase Dashboard usage, the pie chart from the Dashboard will get added to the html report. Additionally, if you set the html report URL to be the same as the Dashboard URL when also using the dashboard, (example: ``--dashboard --html=dashboard.html``), then the Dashboard will become an advanced html report when all the tests complete.
+
+If viewing pytest html reports in [Jenkins](https://www.jenkins.io/), you may need to [configure Jenkins settings](https://stackoverflow.com/a/46197356) for the html to render correctly. This is due to [Jenkins CSP changes](https://www.jenkins.io/doc/book/system-administration/security/configuring-content-security-policy/).
+
 You can also use ``--junit-xml=report.xml`` to get an xml report instead. Jenkins can use this file to display better reporting for your tests.
 
 ```bash

examples/example_logs/ReadMe.md

@@ -23,7 +23,11 @@ nosetests test_fail.py --browser=firefox
 
 <h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> The SeleniumBase Dashboard:</h3>
 
-The ``--dashboard`` option for pytest generates a SeleniumBase Dashboard located at ``dashboard.html``, which updates automatically as tests run and produce results.
+The ``--dashboard`` option for pytest generates a SeleniumBase Dashboard located at ``dashboard.html``, which updates automatically as tests run and produce results. Example:
+
+```bash
+pytest --dashboard --rs --headless
+```
 
 <img src="https://seleniumbase.io/cdn/img/dashboard_1.png" alt="The SeleniumBase Dashboard" title="The SeleniumBase Dashboard" width="360" />
 
@@ -33,7 +37,7 @@ Additionally, you can host your own SeleniumBase Dashboard Server on a port of y
 python -m http.server 1948
 ```
 
-Now you can navigate to ``http://localhost:1948/dashboard.html`` in order to view the served dashboard from a web browser. (Be sure to run that command in the same directory where you ran your tests.)
+Now you can navigate to ``http://localhost:1948/dashboard.html`` in order to view the dashboard as a web app. This requires two different terminal windows: one for running the server, and another for running the tests, which should be run from the same directory.
 
 Here's a full example of what the SeleniumBase Dashboard may look like:
 
@@ -55,8 +59,16 @@ pytest test_suite.py --html=report.html
 
 <img src="https://seleniumbase.io/cdn/img/html_report.png" alt="Example Pytest Report" title="Example Pytest Report" width="520" />
 
+When combining pytest html reports with SeleniumBase Dashboard usage, the pie chart from the Dashboard will get added to the html report. Additionally, if you set the html report URL to be the same as the Dashboard URL when also using the dashboard, (example: ``--dashboard --html=dashboard.html``), then the Dashboard will become an advanced html report when all the tests complete.
+
 If viewing pytest html reports in [Jenkins](https://www.jenkins.io/), you may need to [configure Jenkins settings](https://stackoverflow.com/a/46197356) for the html to render correctly. This is due to [Jenkins CSP changes](https://www.jenkins.io/doc/book/system-administration/security/configuring-content-security-policy/).
 
+You can also use ``--junit-xml=report.xml`` to get an xml report instead. Jenkins can use this file to display better reporting for your tests.
+
+```bash
+pytest test_suite.py --junit-xml=report.xml
+```
+
 --------
 
 <h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> Nosetest Reports:</h3>

help_docs/customizing_test_runs.md

@@ -260,7 +260,11 @@ The code above will leave your browser window open in case there's a failure. (i
 
 <h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> The SeleniumBase Dashboard:</h3>
 
-The ``--dashboard`` option for pytest generates a SeleniumBase Dashboard located at ``dashboard.html``, which updates automatically as tests run and produce results.
+The ``--dashboard`` option for pytest generates a SeleniumBase Dashboard located at ``dashboard.html``, which updates automatically as tests run and produce results. Example:
+
+```bash
+pytest --dashboard --rs --headless
+```
 
 <img src="https://seleniumbase.io/cdn/img/dashboard_1.png" alt="The SeleniumBase Dashboard" title="The SeleniumBase Dashboard" width="360" />
 
@@ -270,7 +274,7 @@ Additionally, you can host your own SeleniumBase Dashboard Server on a port of y
 python -m http.server 1948
 ```
 
-Now you can navigate to ``http://localhost:1948/dashboard.html`` in order to view the served dashboard from a web browser. (Be sure to run that command in the same directory where you ran your tests.)
+Now you can navigate to ``http://localhost:1948/dashboard.html`` in order to view the dashboard as a web app. This requires two different terminal windows: one for running the server, and another for running the tests, which should be run from the same directory.
 
 Here's a full example of what the SeleniumBase Dashboard may look like:
 
@@ -282,7 +286,7 @@ pytest test_suite.py --dashboard --rs --headless
 
 --------
 
-<h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="28" /> Pytest Reports:</h3>
+<h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="32" /> Pytest Reports:</h3>
 
 Using ``--html=report.html`` gives you a fancy report of the name specified after your test suite completes.
 
@@ -292,6 +296,16 @@ pytest test_suite.py --html=report.html
 
 <img src="https://seleniumbase.io/cdn/img/html_report.png" alt="Example Pytest Report" title="Example Pytest Report" width="520" />
 
+When combining pytest html reports with SeleniumBase Dashboard usage, the pie chart from the Dashboard will get added to the html report. Additionally, if you set the html report URL to be the same as the Dashboard URL when also using the dashboard, (example: ``--dashboard --html=dashboard.html``), then the Dashboard will become an advanced html report when all the tests complete.
+
+If viewing pytest html reports in [Jenkins](https://www.jenkins.io/), you may need to [configure Jenkins settings](https://stackoverflow.com/a/46197356) for the html to render correctly. This is due to [Jenkins CSP changes](https://www.jenkins.io/doc/book/system-administration/security/configuring-content-security-policy/).
+
+You can also use ``--junit-xml=report.xml`` to get an xml report instead. Jenkins can use this file to display better reporting for your tests.
+
+```bash
+pytest test_suite.py --junit-xml=report.xml
+```
+
 --------
 
 <h3><img src="https://seleniumbase.io/img/logo6.png" title="SeleniumBase" width="28" /> Nosetest Reports:</h3>

requirements.txt

@@ -72,7 +72,7 @@ toml==0.10.2
 Pillow==6.2.2;python_version<"3.5"
 Pillow==7.2.0;python_version>="3.5" and python_version<"3.6"
 Pillow==8.0.1;python_version>="3.6"
-rich==9.4.0;python_version>="3.6" and python_version<"4.0"
+rich==9.5.1;python_version>="3.6" and python_version<"4.0"
 zipp==1.2.0;python_version<"3.6"
 zipp==3.4.0;python_version>="3.6"
 flake8==3.7.9;python_version<"3.5"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "1.51.0"
+__version__ = "1.51.1"

seleniumbase/fixtures/base_case.py

@@ -6735,6 +6735,11 @@ def __process_dashboard(self, has_exception, init=False):
         if num_untested > 0:
             # Refresh every X seconds when waiting for more test results
             auto_refresh_html = constants.Dashboard.META_REFRESH_HTML
+        else:
+            # The tests are complete
+            if sb_config._using_html_report:
+                # Add the pie chart to the pytest html report
+                sb_config._saved_dashboard_pie = self.extract_chart()
         head = (
             '<head><meta charset="utf-8" />'
             '<meta property="og:image" '
@@ -6832,6 +6837,8 @@ def __process_dashboard(self, has_exception, init=False):
         refresh_line = (
             '<script type="text/javascript" src="%s">'
             '</script>' % constants.Dashboard.LIVE_JS)
+        if num_untested == 0 and sb_config._using_html_report:
+            sb_config._dash_final_summary = status
         add_more = add_more + refresh_line
         the_html = head + self.extract_chart() + table_html + add_more
         abs_path = os.path.abspath('.')

seleniumbase/plugins/pytest_plugin.py

@@ -594,10 +594,21 @@ def pytest_configure(config):
     sb_config._only_unittest = True  # If any test uses BaseCase, becomes False
     sb_config._sbase_detected = False  # Becomes True during SeleniumBase tests
     sb_config._extra_dash_entries = []  # Dashboard entries for non-SBase tests
+    sb_config._using_html_report = False  # Becomes True when using html report
+    sb_config._dash_is_html_report = False  # Dashboard becomes the html report
+    sb_config._saved_dashboard_pie = None  # Copy of pie chart for html report
+    sb_config._dash_final_summary = None  # Dash status to add to html report
+    sb_config._html_report_name = None  # The name of the pytest html report
 
     arg_join = " ".join(sys.argv)
     if ("-n" in sys.argv) or ("-n=" in arg_join):
         sb_config._multithreaded = True
+    if ("--html" in sys.argv or "--html=" in arg_join):
+        sb_config._using_html_report = True
+        sb_config._html_report_name = config.getoption("htmlpath")
+        if sb_config.dashboard:
+            if sb_config._html_report_name == "dashboard.html":
+                sb_config._dash_is_html_report = True
 
     if "linux" in sys.platform and (
             not sb_config.headed and not sb_config.headless):
@@ -654,10 +665,16 @@ def pytest_runtest_setup():
         if not sb_config.item_count_finalized:
             sb_config.item_count_finalized = True
             sb_config.item_count_untested = sb_config.item_count
-            dashboard_html = "file://" + os.getcwd() + "/dashboard.html"
+            dashboard_html = os.getcwd() + "/dashboard.html"
             star_len = len("Dashboard: ") + len(dashboard_html)
             stars = "*" * star_len
-            print("\nDashboard: %s\n%s" % (dashboard_html, stars))
+            colorama.init(autoreset=True)
+            c1 = colorama.Fore.BLUE + colorama.Back.LIGHTYELLOW_EX
+            cr = colorama.Style.RESET_ALL
+            if "linux" in sys.platform:
+                c1 = ''
+                cr = ''
+            print("\nDashboard: %s%s%s\n%s" % (c1, dashboard_html, cr, stars))
 
 
 def pytest_runtest_teardown(item):
@@ -687,6 +704,20 @@ def pytest_sessionfinish(session):
     pass
 
 
+def pytest_terminal_summary(terminalreporter):
+    if sb_config._has_exception and (
+            sb_config.dashboard and not sb_config._only_unittest):
+        # Print link a second time because the first one may be off-screen
+        dashboard_file = os.getcwd() + "/dashboard.html"
+        terminalreporter.write_sep(
+            "-", "Dashboard: %s" % dashboard_file)
+    if sb_config._has_exception or sb_config.save_screenshot:
+        # Log files are generated during test failures and Screenshot Mode
+        latest_logs_dir = os.getcwd() + "/latest_logs/"
+        terminalreporter.write_sep(
+            "-", "LogPath: %s" % latest_logs_dir)
+
+
 def pytest_unconfigure():
     """ This runs after all tests have completed with pytest. """
     proxy_helper.remove_proxy_zip_if_present()
@@ -702,12 +733,20 @@ def pytest_unconfigure():
         sb_config.shared_driver = None
     log_helper.archive_logs_if_set(sb_config.log_path, sb_config.archive_logs)
 
-    # Dashboard post-processing: Disable auto-refresh and stamp complete
+    # Dashboard post-processing: Disable time-based refresh and stamp complete
     if sb_config.dashboard and not sb_config._only_unittest:
-        stamp = "\n<!--Test Run Complete-->"
+        stamp = ""
+        if sb_config._dash_is_html_report:
+            # (If the Dashboard URL is the same as the HTML Report URL:)
+            # Have the html report refresh back to a dashboard on update
+            stamp += (
+                '\n<script type="text/javascript" src="%s">'
+                '</script>' % constants.Dashboard.LIVE_JS)
+        stamp += "\n<!--Test Run Complete-->"
         find_it = constants.Dashboard.META_REFRESH_HTML
         swap_with = ''  # Stop refreshing the page after the run is done
         try:
+            # Part 1: Finalizing the dashboard / integrating html report
             time.sleep(0.3)  # Add time for "livejs" to detect changes
             abs_path = os.path.abspath('.')
             dashboard_path = os.path.join(abs_path, "dashboard.html")
@@ -717,27 +756,43 @@ def pytest_unconfigure():
                 # If the test run doesn't complete by itself, stop refresh
                 the_html = the_html.replace(find_it, swap_with)
                 the_html += stamp
+                if sb_config._dash_is_html_report and (
+                        sb_config._saved_dashboard_pie):
+                    the_html = the_html.replace(
+                        "<h1>dashboard.html</h1>",
+                        sb_config._saved_dashboard_pie)
+                    the_html = the_html.replace(
+                        "</head>", '</head><link rel="shortcut icon" '
+                        'href="https://seleniumbase.io/img/dash_pie_2.png">')
+                    if sb_config._dash_final_summary:
+                        the_html += sb_config._dash_final_summary
                 time.sleep(0.25)
                 with open(dashboard_path, "w", encoding='utf-8') as f:
                     f.write(the_html)
+            # Part 2: Appending a pytest html report with dashboard data
+            html_report_path = os.path.join(
+                abs_path, sb_config._html_report_name)
+            if sb_config._using_html_report and (
+                    os.path.exists(html_report_path) and
+                    not sb_config._dash_is_html_report):
+                # Add the dashboard pie to the pytest html report
+                with open(html_report_path, 'r', encoding='utf-8') as f:
+                    the_html = f.read()
+                if sb_config._saved_dashboard_pie:
+                    the_html = the_html.replace(
+                        "<h1>%s</h1>" % sb_config._html_report_name,
+                        sb_config._saved_dashboard_pie)
+                    the_html = the_html.replace(
+                        "</head>", '</head><link rel="shortcut icon" '
+                        'href="https://seleniumbase.io/img/dash_pie_2.png">')
+                    if sb_config._dash_final_summary:
+                        the_html += sb_config._dash_final_summary
+                with open(html_report_path, "w", encoding='utf-8') as f:
+                    f.write(the_html)
         except Exception:
             pass
 
 
-def pytest_terminal_summary(terminalreporter):
-    if sb_config._has_exception and (
-            sb_config.dashboard and not sb_config._only_unittest):
-        # Print link a second time because the first one may be off-screen
-        dashboard_file = "file://" + os.getcwd() + "/dashboard.html"
-        terminalreporter.write_sep(
-            "-", "Dashboard: %s" % dashboard_file)
-    if sb_config._has_exception or sb_config.save_screenshot:
-        # Log files are generated during test failures and Screenshot Mode
-        latest_logs_dir = os.getcwd() + "/latest_logs/"
-        terminalreporter.write_sep(
-            "-", "Log files: %s" % latest_logs_dir)
-
-
 @pytest.fixture()
 def sb(request):
     """ SeleniumBase as a pytest fixture.
@@ -827,5 +882,12 @@ def pytest_runtest_makereport(item, call):
             extra = getattr(report, 'extra', [])
             if len(extra_report) > 1 and extra_report[1]["content"]:
                 report.extra = extra + extra_report
+            if sb_config._dash_is_html_report:
+                # (If the Dashboard URL is the same as the HTML Report URL:)
+                # Have the html report refresh back to a dashboard on update
+                refresh_updates = (
+                    '<script type="text/javascript" src="%s">'
+                    '</script>' % constants.Dashboard.LIVE_JS)
+                report.extra.append(pytest_html.extras.html(refresh_updates))
         except Exception:
             pass

setup.py

@@ -176,7 +176,7 @@
         'Pillow==6.2.2;python_version<"3.5"',
         'Pillow==7.2.0;python_version>="3.5" and python_version<"3.6"',
         'Pillow==8.0.1;python_version>="3.6"',
-        'rich==9.4.0;python_version>="3.6" and python_version<"4.0"',
+        'rich==9.5.1;python_version>="3.6" and python_version<"4.0"',
         'zipp==1.2.0;python_version<"3.6"',
         'zipp==3.4.0;python_version>="3.6"',
         'flake8==3.7.9;python_version<"3.5"',