Merge pull request #1034 from seleniumbase/updates-to-visual-testing

Some updates to Visual Testing

Author: mdmintz

README.md

@@ -7,7 +7,7 @@
 
 <h3 align="center"><a href="https://github.com/seleniumbase/SeleniumBase/"><img src="https://seleniumbase.io/cdn/img/mac_sb_logo_5.png" alt="SeleniumBase" title="SeleniumBase" width="300" /></a></h3>
 <!-- View on GitHub -->
-<h4 align="center">Better web testing with <a href="https://www.selenium.dev/documentation/">Selenium</a> and <a href="https://docs.pytest.org/en/latest/index.html">pytest</a>.</h4>
+<h4 align="center">Better web testing with <a href="https://www.selenium.dev/documentation/">Selenium</a> and <a href="https://docs.pytest.org/en/stable/">pytest</a>.</h4>
 <p align="center">
 <a href="https://github.com/seleniumbase/SeleniumBase/releases">
 <img src="https://img.shields.io/github/v/release/seleniumbase/SeleniumBase.svg?color=2277EE" alt="Latest Release on GitHub" /></a> <a href="https://pypi.python.org/pypi/seleniumbase">
@@ -84,8 +84,9 @@ pip install -e .  # Editable install
 pip install seleniumbase
 ```
 
-> (Add ``--upgrade`` OR ``-U`` to upgrade an installation.)
-> (Add ``--force-reinstall`` to upgrade dependencies.)
+> (Add ``--upgrade`` OR ``-U`` to install the latest ``seleniumbase``.)
+> (Add ``--force-reinstall`` to also install the latest dependencies.)
+> (Add ``--index-url=http://pypi.python.org/simple/`` if blocked by a VPN.)
 > (Use ``pip3`` if multiple versions of Python are installed.)
 
 🔵 Type ``seleniumbase`` or ``sbase`` to verify that SeleniumBase was installed successfully:

help_docs/method_summary.md

@@ -672,7 +672,8 @@ self.switch_to_alert(timeout=None)
 
 ############
 
-self.check_window(name="default", level=0, baseline=False, check_domain=True)
+self.check_window(
+    name="default", level=0, baseline=False, check_domain=True, full_diff=False)
 
 ############
 
@@ -682,6 +683,12 @@ self.deferred_assert_element(selector, by=By.CSS_SELECTOR, timeout=None)
 self.deferred_assert_text(text, selector="html", by=By.CSS_SELECTOR, timeout=None)
 # Duplicates: self.delayed_assert_text(text, selector="html", by=By.CSS_SELECTOR, timeout=None)
 
+self.deferred_check_window(
+    name="default", level=0, baseline=False, check_domain=True, full_diff=False)
+# Duplicates:
+# self.deferred_check_window(
+#     name=name, level=level, baseline=baseline, check_domain=check_domain, full_diff=full_diff)
+
 self.process_deferred_asserts(print_only=False)
 # Duplicates: self.process_delayed_asserts(print_only=False)
 

mkdocs_build/requirements.txt

@@ -1,12 +1,12 @@
 regex>=2021.10.23
 tqdm>=4.62.3
+docutils==0.17.1
+python-dateutil==2.8.2
 livereload==2.6.3;python_version>="3.6"
 joblib==1.1.0;python_version>="3.6"
 Markdown==3.3.4;python_version>="3.6"
 MarkupSafe==2.0.1;python_version>="3.6"
-jsmin==3.0.0;python_version>="3.6"
-docutils==0.17.1
-python-dateutil==2.8.2
+pyparsing==3.0.1;python_version>="3.6"
 keyring==23.2.1;python_version>="3.6"
 pkginfo==1.7.1;python_version>="3.6"
 Jinja2==3.0.2;python_version>="3.6"
@@ -16,6 +16,7 @@ readme-renderer==30.0
 pymdown-extensions==9.0;python_version>="3.6"
 importlib-metadata==4.8.1;python_version>="3.6"
 bleach==4.1.0
+jsmin==3.0.0;python_version>="3.6"
 lunr==0.6.1;python_version>="3.6"
 nltk==3.6.5;python_version>="3.6"
 watchdog==2.1.6;python_version>="3.6"

requirements.txt

@@ -8,7 +8,7 @@ setuptools>=50.3.2;python_version>="3.5" and python_version<"3.6"
 setuptools>=58.3.0;python_version>="3.6"
 setuptools-scm==5.0.2;python_version<"3.6"
 setuptools-scm>=6.3.2;python_version>="3.6"
-tomli>=1.2.1;python_version>="3.6"
+tomli>=1.2.2;python_version>="3.6"
 wheel>=0.37.0
 attrs>=21.2.0
 PyYAML>=6.0;python_version>="3.6"
@@ -80,7 +80,7 @@ cryptography==35.0.0;python_version>="3.7"
 pygments==2.5.2;python_version<"3.5"
 pygments==2.10.0;python_version>="3.5"
 traitlets==4.3.3;python_version<"3.7"
-traitlets==5.1.0;python_version>="3.7"
+traitlets==5.1.1;python_version>="3.7"
 prompt-toolkit==1.0.18;python_version<"3.5"
 prompt-toolkit==2.0.10;python_version>="3.5" and python_version<"3.6.2"
 prompt-toolkit==3.0.21;python_version>="3.6.2"

seleniumbase/__version__.py

@@ -1,2 +1,2 @@
 # seleniumbase package
-__version__ = "2.0.6"
+__version__ = "2.0.7"

seleniumbase/fixtures/base_case.py

@@ -9292,7 +9292,12 @@ def __assert_eq(self, *args, **kwargs):
             raise Exception(minified_exception)
 
     def check_window(
-        self, name="default", level=0, baseline=False, check_domain=True
+        self,
+        name="default",
+        level=0,
+        baseline=False,
+        check_domain=True,
+        full_diff=False,
     ):
         """***  Automated Visual Testing with SeleniumBase  ***
 
@@ -9351,6 +9356,10 @@ def check_window(
         only makes sense if you're calling self.check_window() more than
         once with the same name parameter in the same test.
 
+        If "full_diff" is set to False, the error output will only
+        include the first differing element in the list comparison.
+        Set "full_diff" to True if you want to see the full output.
+
         Automated Visual Testing with self.check_window() is not very
         effective for websites that have dynamic content that changes
         the layout and structure of web pages. For those, you're much
@@ -9482,12 +9491,21 @@ def check_window(
                 self.assertEqual(page_data_domain, page_domain, domain_fail)
             unittest.TestCase.maxDiff = 6400  # Use `None` for no limit
             if level == 3:
-                self.__assert_eq(level_3_data, level_3, level_3_failure)
+                if not full_diff:
+                    self.__assert_eq(level_3_data, level_3, level_3_failure)
+                else:
+                    self.assertEqual(level_3_data, level_3, level_3_failure)
             unittest.TestCase.maxDiff = 3200
             if level == 2:
-                self.__assert_eq(level_2_data, level_2, level_2_failure)
+                if not full_diff:
+                    self.__assert_eq(level_2_data, level_2, level_2_failure)
+                else:
+                    self.assertEqual(level_2_data, level_2, level_2_failure)
             if level == 1:
-                self.__assert_eq(level_1_data, level_1, level_1_failure)
+                if not full_diff:
+                    self.__assert_eq(level_1_data, level_1, level_1_failure)
+                else:
+                    self.assertEqual(level_1_data, level_1, level_1_failure)
             unittest.TestCase.maxDiff = 6400  # Use `None` for no limit
             if level == 0:
                 try:
@@ -9497,19 +9515,36 @@ def check_window(
                             page_domain, page_data_domain, domain_fail
                         )
                     try:
-                        self.__assert_eq(
-                            level_1_data, level_1, level_1_failure
-                        )
+                        if not full_diff:
+                            self.__assert_eq(
+                                level_1_data, level_1, level_1_failure
+                            )
+                        else:
+                            self.assertEqual(
+                                level_1_data, level_1, level_1_failure
+                            )
                     except Exception as e:
                         print(e)
                     try:
-                        self.__assert_eq(
-                            level_2_data, level_2, level_2_failure
-                        )
+                        if not full_diff:
+                            self.__assert_eq(
+                                level_2_data, level_2, level_2_failure
+                            )
+                        else:
+                            self.assertEqual(
+                                level_2_data, level_2, level_2_failure
+                            )
                     except Exception as e:
                         print(e)
                     unittest.TestCase.maxDiff = 6400  # Use `None` for no limit
-                    self.__assert_eq(level_3_data, level_3, level_3_failure)
+                    if not full_diff:
+                        self.__assert_eq(
+                            level_3_data, level_3, level_3_failure
+                        )
+                    else:
+                        self.assertEqual(
+                            level_3_data, level_3, level_3_failure
+                        )
                 except Exception as e:
                     print(e)  # Level-0 Dry Run (Only print the differences)
             unittest.TestCase.maxDiff = None  # Reset unittest.TestCase.maxDiff
@@ -9635,6 +9670,32 @@ def deferred_assert_text(
             self.__add_deferred_assert_failure()
             return False
 
+    def deferred_check_window(
+        self,
+        name="default",
+        level=0,
+        baseline=False,
+        check_domain=True,
+        full_diff=False,
+    ):
+        """A non-terminating assertion for the check_window() method.
+        Failures will be saved until the process_deferred_asserts()
+        method is called from inside a test, likely at the end of it."""
+        self.__check_scope()
+        self.__deferred_assert_count += 1
+        try:
+            self.check_window(
+                name=name,
+                level=level,
+                baseline=baseline,
+                check_domain=check_domain,
+                full_diff=full_diff,
+            )
+            return True
+        except Exception:
+            self.__add_deferred_assert_failure()
+            return False
+
     def process_deferred_asserts(self, print_only=False):
         """To be used with any test that uses deferred_asserts, which are
         non-terminating verifications that only raise exceptions
@@ -9657,7 +9718,7 @@ def process_deferred_asserts(self, print_only=False):
             if print_only:
                 print(exception_output)
             else:
-                raise Exception(exception_output)
+                raise Exception(exception_output.replace("\\n", "\n"))
 
     ############
 
@@ -9679,6 +9740,23 @@ def delayed_assert_text(
             text=text, selector=selector, by=by, timeout=timeout
         )
 
+    def delayed_check_window(
+        self,
+        name="default",
+        level=0,
+        baseline=False,
+        check_domain=True,
+        full_diff=False
+    ):
+        """ Same as self.deferred_check_window() """
+        return self.deferred_check_window(
+            name=name,
+            level=level,
+            baseline=baseline,
+            check_domain=check_domain,
+            full_diff=full_diff,
+        )
+
     def process_delayed_asserts(self, print_only=False):
         """ Same as self.process_deferred_asserts() """
         self.process_deferred_asserts(print_only=print_only)
@@ -10743,7 +10821,6 @@ def __create_log_path_as_needed(self, test_logpath):
 
     def __process_dashboard(self, has_exception, init=False):
         """ SeleniumBase Dashboard Processing """
-        existing_res = sb_config._results  # Used by multithreaded tests
         if self._multithreaded:
             abs_path = os.path.abspath(".")
             dash_json_loc = constants.Dashboard.DASH_JSON
@@ -10831,8 +10908,8 @@ def __process_dashboard(self, has_exception, init=False):
                 sb_config.item_count_untested -= 1
             elif (
                 self._multithreaded
-                and test_id in existing_res.keys()
-                and existing_res[test_id] == "Skipped"
+                and test_id in sb_config._results.keys()
+                and sb_config._results[test_id] == "Skipped"
             ):
                 sb_config._results[test_id] = "Skipped"
                 sb_config.item_count_skipped += 1

seleniumbase/plugins/pytest_plugin.py

@@ -1483,7 +1483,7 @@ def _perform_pytest_unconfigure_():
         pass
 
 
-def pytest_unconfigure():
+def pytest_unconfigure(config):
     """ This runs after all tests have completed with pytest. """
     if hasattr(sb_config, "_multithreaded") and sb_config._multithreaded:
         import fasteners
@@ -1492,7 +1492,6 @@ def pytest_unconfigure():
         if (
             hasattr(sb_config, "dashboard")
             and sb_config.dashboard
-            and sb_config._dash_html
         ):
             # Multi-threaded tests with the Dashboard
             abs_path = os.path.abspath(".")
@@ -1502,17 +1501,29 @@ def pytest_unconfigure():
                 sb_config._only_unittest = False
                 dashboard_path = os.path.join(abs_path, "dashboard.html")
                 with dash_lock:
-                    with open(dashboard_path, "w", encoding="utf-8") as f:
-                        f.write(sb_config._dash_html)
+                    if (
+                        sb_config._dash_html
+                        and config.getoption("htmlpath") == "dashboard.html"
+                    ):
+                        # Dash is HTML Report (Multithreaded)
+                        sb_config._dash_is_html_report = True
+                        with open(dashboard_path, "w", encoding="utf-8") as f:
+                            f.write(sb_config._dash_html)
+                    # Dashboard Multithreaded
                     _perform_pytest_unconfigure_()
                     return
+            else:
+                # Dash Lock is missing
+                _perform_pytest_unconfigure_()
+                return
         with dash_lock:
             # Multi-threaded tests
             _perform_pytest_unconfigure_()
             return
     else:
         # Single-threaded tests
         _perform_pytest_unconfigure_()
+        return
 
 
 @pytest.fixture()

setup.py

@@ -124,7 +124,7 @@
         'setuptools>=58.3.0;python_version>="3.6"',
         'setuptools-scm==5.0.2;python_version<"3.6"',
         'setuptools-scm>=6.3.2;python_version>="3.6"',
-        'tomli>=1.2.1;python_version>="3.6"',
+        'tomli>=1.2.2;python_version>="3.6"',
         "wheel>=0.37.0",
         "attrs>=21.2.0",
         'PyYAML>=6.0;python_version>="3.6"',
@@ -196,7 +196,7 @@
         'pygments==2.5.2;python_version<"3.5"',
         'pygments==2.10.0;python_version>="3.5"',
         'traitlets==4.3.3;python_version<"3.7"',
-        'traitlets==5.1.0;python_version>="3.7"',
+        'traitlets==5.1.1;python_version>="3.7"',
         'prompt-toolkit==1.0.18;python_version<"3.5"',
         'prompt-toolkit==2.0.10;python_version>="3.5" and python_version<"3.6.2"',  # noqa: E501
         'prompt-toolkit==3.0.21;python_version>="3.6.2"',