Add check_window() for making visual comparison validations

Author: mdmintz

seleniumbase/fixtures/base_case.py

@@ -21,6 +21,8 @@ def test_anything(self):
 Code becomes greatly simplified and easier to maintain.
 """
 
+import codecs
+import json
 import logging
 import math
 import os
@@ -38,6 +40,7 @@ def test_anything(self):
 from seleniumbase.core import download_helper
 from seleniumbase.core import log_helper
 from seleniumbase.core import tour_helper
+from seleniumbase.core import visual_helper
 from seleniumbase.fixtures import constants
 from seleniumbase.fixtures import js_utils
 from seleniumbase.fixtures import page_actions
@@ -2436,6 +2439,198 @@ def switch_to_window(self, window, timeout=settings.SMALL_TIMEOUT):
     def switch_to_default_window(self):
         self.switch_to_window(0)
 
+    def check_window(self, name="default", level=0, baseline=False):
+        """ ***  Automated Visual Testing with SeleniumBase  ***
+
+            The first time a test calls self.check_window() for a unique "name"
+            parameter provided, it will set a visual baseline, meaning that it
+            creates a folder, saves the URL to a file, saves the current window
+            screenshot to a file, and creates the following three files
+            with the listed data saved:
+            tags_level1.txt  ->  HTML tags from the window
+            tags_level2.txt  ->  HTML tags + attributes from the window
+            tags_level3.txt  ->  HTML tags + attributes/values from the window
+
+            Baseline folders are named based on the test name and the name
+            parameter passed to self.check_window(). The same test can store
+            multiple baseline folders.
+
+            If the baseline is being set/reset, the "level" doesn't matter.
+
+            After the first run of self.check_window(), it will compare the
+            HTML tags of the latest window to the one from the initial run.
+            Here's how the level system works:
+            * level=0 ->
+                DRY RUN ONLY - Will perform a comparison to the baseline, and
+                               print out any differences that are found, but
+                               won't fail the test even if differences exist.
+            * level=1 ->
+                HTML tags are compared to tags_level1.txt
+            * level=2 ->
+                HTML tags are compared to tags_level1.txt and
+                HTML tags/attributes are compared to tags_level2.txt
+            * level=3 ->
+                HTML tags are compared to tags_level1.txt and
+                HTML tags + attributes are compared to tags_level2.txt and
+                HTML tags + attributes/values are compared to tags_level3.txt
+            As shown, Level-3 is the most strict, Level-1 is the least strict.
+            If the comparisons from the latest window to the existing baseline
+            don't match, the current test will fail, except for Level-0 tests.
+
+            You can reset the visual baseline on the command line by using:
+                --visual_baseline
+            As long as "--visual_baseline" is used on the command line while
+            running tests, the self.check_window() method cannot fail because
+            it will rebuild the visual baseline rather than comparing the html
+            tags of the latest run to the existing baseline. If there are any
+            expected layout changes to a website that you're testing, you'll
+            need to reset the baseline to prevent unnecessary failures.
+
+            self.check_window() will fail with "Page Domain Mismatch Failure"
+            if the page domain doesn't match the domain of the baseline.
+
+            If you want to use self.check_window() to compare a web page to
+            a later version of itself from within the same test run, you can
+            add the parameter "baseline=True" to the first time you call
+            self.check_window() in a test to use that as the baseline. This
+            only makes sense if you're calling self.check_window() more than
+            once with the same name parameter in the same test.
+
+            Automated Visual Testing with self.check_window() is not very
+            effective for websites that have dynamic content that changes
+            the layout and structure of web pages. For those, you're much
+            better off using regular SeleniumBase functional testing.
+
+            Example usage:
+                self.check_window(name="testing", level=0)
+                self.check_window(name="xkcd_home", level=1)
+                self.check_window(name="github_page", level=2)
+                self.check_window(name="wikipedia_page", level=3)
+        """
+        if level == "0":
+            level = 0
+        if level == "1":
+            level = 1
+        if level == "2":
+            level = 2
+        if level == "3":
+            level = 3
+        if level != 0 and level != 1 and level != 2 and level != 3:
+            raise Exception('Parameter "level" must be set to 0, 1, 2, or 3!')
+
+        module = self.__class__.__module__
+        if '.' in module and len(module.split('.')[-1]) > 1:
+            module = module.split('.')[-1]
+        test_id = "%s.%s" % (module, self._testMethodName)
+        if not name or len(name) < 1:
+            name = "default"
+        name = str(name)
+        visual_helper.visual_baseline_folder_setup()
+        baseline_dir = constants.VisualBaseline.STORAGE_FOLDER
+        visual_baseline_path = baseline_dir + "/" + test_id + "/" + name
+        page_url_file = visual_baseline_path + "/page_url.txt"
+        screenshot_file = visual_baseline_path + "/screenshot.png"
+        level_1_file = visual_baseline_path + "/tags_level_1.txt"
+        level_2_file = visual_baseline_path + "/tags_level_2.txt"
+        level_3_file = visual_baseline_path + "/tags_level_3.txt"
+
+        set_baseline = False
+        if baseline or self.visual_baseline:
+            set_baseline = True
+        if not os.path.exists(visual_baseline_path):
+            set_baseline = True
+            try:
+                os.makedirs(visual_baseline_path)
+            except Exception:
+                pass  # Only reachable during multi-threaded test runs
+        if not os.path.exists(page_url_file):
+            set_baseline = True
+        if not os.path.exists(screenshot_file):
+            set_baseline = True
+        if not os.path.exists(level_1_file):
+            set_baseline = True
+        if not os.path.exists(level_2_file):
+            set_baseline = True
+        if not os.path.exists(level_3_file):
+            set_baseline = True
+
+        page_url = self.get_current_url()
+        soup = self.get_beautiful_soup()
+        html_tags = soup.body.find_all()
+        level_1 = [[tag.name] for tag in html_tags]
+        level_1 = json.loads(json.dumps(level_1))  # Tuples become lists
+        level_2 = [[tag.name, sorted(tag.attrs.keys())] for tag in html_tags]
+        level_2 = json.loads(json.dumps(level_2))  # Tuples become lists
+        level_3 = [[tag.name, sorted(tag.attrs.items())] for tag in html_tags]
+        level_3 = json.loads(json.dumps(level_3))  # Tuples become lists
+
+        if set_baseline:
+            self.save_screenshot("screenshot.png", visual_baseline_path)
+            out_file = codecs.open(page_url_file, "w+")
+            out_file.writelines(page_url)
+            out_file.close()
+            out_file = codecs.open(level_1_file, "w+")
+            out_file.writelines(json.dumps(level_1))
+            out_file.close()
+            out_file = codecs.open(level_2_file, "w+")
+            out_file.writelines(json.dumps(level_2))
+            out_file.close()
+            out_file = codecs.open(level_3_file, "w+")
+            out_file.writelines(json.dumps(level_3))
+            out_file.close()
+
+        if not set_baseline:
+            f = open(page_url_file, 'r')
+            page_url_data = f.read().strip()
+            f.close()
+            f = open(level_1_file, 'r')
+            level_1_data = json.loads(f.read())
+            f.close()
+            f = open(level_2_file, 'r')
+            level_2_data = json.loads(f.read())
+            f.close()
+            f = open(level_3_file, 'r')
+            level_3_data = json.loads(f.read())
+            f.close()
+
+            domain_fail = (
+                "Page Domain Mismatch Failure: "
+                "Current Page Domain doesn't match the Page Domain of the "
+                "Baseline! Can't compare two completely different sites! "
+                "Run with --visual_baseline to reset the baseline!")
+            level_1_failure = (
+                "\n\n*** Exception: <Level 1> Visual Diff Failure:\n"
+                "* HTML tags don't match the baseline!")
+            level_2_failure = (
+                "\n\n*** Exception: <Level 2> Visual Diff Failure:\n"
+                "* HTML tag attributes don't match the baseline!")
+            level_3_failure = (
+                "\n\n*** Exception: <Level 3> Visual Diff Failure:\n"
+                "* HTML tag attribute values don't match the baseline!")
+
+            page_domain = self.get_domain_url(page_url)
+            page_data_domain = self.get_domain_url(page_url_data)
+            unittest.TestCase.maxDiff = 1000
+            if level == 1 or level == 2 or level == 3:
+                self.assert_equal(page_domain, page_data_domain, domain_fail)
+                self.assert_equal(level_1, level_1_data, level_1_failure)
+            unittest.TestCase.maxDiff = None
+            if level == 2 or level == 3:
+                self.assert_equal(level_2, level_2_data, level_2_failure)
+            if level == 3:
+                self.assert_equal(level_3, level_3_data, level_3_failure)
+            if level == 0:
+                try:
+                    unittest.TestCase.maxDiff = 1000
+                    self.assert_equal(
+                        page_domain, page_data_domain, domain_fail)
+                    self.assert_equal(level_1, level_1_data, level_1_failure)
+                    unittest.TestCase.maxDiff = None
+                    self.assert_equal(level_2, level_2_data, level_2_failure)
+                    self.assert_equal(level_3, level_3_data, level_3_failure)
+                except Exception as e:
+                    print(e)  # Level-0 Dry Run (Only print the differences)
+
     def save_screenshot(self, name, folder=None):
         """ The screenshot will be in PNG format. """
         return page_actions.save_screenshot(self.driver, name, folder)
@@ -2888,6 +3083,7 @@ def setUp(self, masterqa_mode=False):
             self.verify_delay = sb_config.verify_delay
             self.disable_csp = sb_config.disable_csp
             self.save_screenshot_after_test = sb_config.save_screenshot
+            self.visual_baseline = sb_config.visual_baseline
             self.timeout_multiplier = sb_config.timeout_multiplier
             self.use_grid = False
             if self.servername != "localhost":