Skip to content

Commit 5190ece

Browse files
mdmintzmdmintz
committed
Expand visual testing logs with a side-by-side comparison
1 parent 5d4e924 commit 5190ece

File tree

3 files changed

+141
-7
lines changed

3 files changed

+141
-7
lines changed

‎seleniumbase/core/encoded_images.py

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -126,3 +126,10 @@
126126
"CtZrdwUAJRk/AoTJspXWuCo61KDQjw2oCBgOwE+egH3kfj7umAhCOmwEi050g4nslK+I4W/5C"
127127
"zyeuQ5f3kZ4tNlIZF18vP/AXYR+dvV3FCCAAAAAElFTkSuQmCC"
128128
)
129+
SIDE_BY_SIDE_PNG = (
130+
"data:image/gif;base64,iVBORw0KGgoAAAANSUhEUgAAABAAAAAQCAIAAACQkWg2AAAAAXN"
131+
"SR0IArs4c6QAAAIVJREFUKBVjvPLvPwMpgIkUxSC1LBANOo0oGq/Ug7hYBUm2gWQNjBBPa/1l"
132+
"RHbTNWZQSGAVhPqBaRWyeoYrkSAuozaKIMMNEJdkJ5GsAeqkf2Eotl8D8/5fRRGEcGB+aEGJb"
133+
"0g8MGEVxGIIXiFy/eC8CuJsmOH1WkAWVkFoxMEUEqZJdhIAo3Aj/iHmzlMAAAAASUVORK5CYI"
134+
"I="
135+
)

‎seleniumbase/fixtures/base_case.py

Lines changed: 127 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@ def test_anything(self):
3535
import logging
3636
import os
3737
import re
38+
import shutil
3839
import sys
3940
import time
4041
import unittest
@@ -92,6 +93,7 @@ def __init__(self, *args, **kwargs):
9293
self.__screenshot_count = 0
9394
self.__will_be_skipped = False
9495
self.__passed_then_skipped = False
96+
self.__visual_baseline_copies = []
9597
self.__last_url_of_deferred_assert = "data:,"
9698
self.__last_page_load_url = "data:,"
9799
self.__last_page_screenshot = None
@@ -9548,6 +9550,90 @@ def __assert_eq(self, *args, **kwargs):
95489550
if minified_exception:
95499551
raise Exception(minified_exception)
95509552

9553+
def __process_visual_baseline_logs(self):
9554+
""" Save copies of baseline PNGs in "./latest_logs" during failures.
9555+
Also create a side_by_side.html file for visual comparisons. """
9556+
test_logpath = os.path.join(self.log_path, self.__get_test_id())
9557+
for baseline_copy_tuple in self.__visual_baseline_copies:
9558+
baseline_path = baseline_copy_tuple[0]
9559+
baseline_copy_path = baseline_copy_tuple[1]
9560+
b_c_alt_path = baseline_copy_tuple[2]
9561+
latest_png_path = baseline_copy_tuple[3]
9562+
latest_copy_path = baseline_copy_tuple[4]
9563+
l_c_alt_path = baseline_copy_tuple[5]
9564+
9565+
if len(self.__visual_baseline_copies) == 1:
9566+
baseline_copy_path = b_c_alt_path
9567+
latest_copy_path = l_c_alt_path
9568+
if (
9569+
os.path.exists(baseline_path)
9570+
and not os.path.exists(baseline_copy_path)
9571+
):
9572+
self.__create_log_path_as_needed(test_logpath)
9573+
shutil.copy(baseline_path, baseline_copy_path)
9574+
if (
9575+
os.path.exists(latest_png_path)
9576+
and not os.path.exists(latest_copy_path)
9577+
):
9578+
self.__create_log_path_as_needed(test_logpath)
9579+
shutil.copy(latest_png_path, latest_copy_path)
9580+
if len(self.__visual_baseline_copies) != 1:
9581+
return # Only possible when deferred visual asserts are used
9582+
head = (
9583+
'<head><meta charset="utf-8">'
9584+
'<meta name="viewport" content="shrink-to-fit=no">'
9585+
'<link rel="shortcut icon" href="%s">'
9586+
"<title>Visual Comparison</title>"
9587+
"</head>"
9588+
% (constants.SideBySide.SIDE_BY_SIDE_PNG)
9589+
)
9590+
table_html = (
9591+
'<table border="3px solid #E6E6E6;" width="100%;" padding: 12px;'
9592+
' font-size="16px;" text-align="left;" id="results-table"'
9593+
' style="background-color: #FAFAFA;">'
9594+
'<thead id="results-table-head">'
9595+
'<tr>'
9596+
'<th style="background-color: rgba(0, 128, 0, 0.25);"'
9597+
' col="baseline">Baseline Screenshot</th>'
9598+
'<th style="background-color: rgba(128, 0, 0, 0.25);"'
9599+
' col="failure">Visual Diff Failure Screenshot</th>'
9600+
"</tr></thead>"
9601+
)
9602+
row = (
9603+
'<tbody class="compare results-table-row">'
9604+
'<tr style="background-color: #F4F4FE;">'
9605+
'<td><img src="%s" width="100%%" /></td>'
9606+
'<td><img src="%s" width="100%%" /></td>'
9607+
"</tr></tbody>"
9608+
"" % ("baseline.png", "baseline_diff.png")
9609+
)
9610+
header_text = "SeleniumBase Visual Comparison"
9611+
header = '<h3 align="center">%s</h3>' % header_text
9612+
table_html += row
9613+
table_html += "</table>"
9614+
footer = "<br /><b>Last updated:</b> "
9615+
timestamp, the_date, the_time = log_helper.get_master_time()
9616+
last_updated = "%s at %s" % (the_date, the_time)
9617+
footer = footer + "%s" % last_updated
9618+
gen_by = (
9619+
'<p><div>Generated by: <b><a href="https://seleniumbase.io/">'
9620+
"SeleniumBase</a></b></div></p><p></p>"
9621+
)
9622+
footer = footer + gen_by
9623+
the_html = (
9624+
'<html lang="en">'
9625+
+ head
9626+
+ '<body style="background-color: #FCFCF4;">'
9627+
+ header
9628+
+ table_html
9629+
+ footer
9630+
+ "</body>"
9631+
)
9632+
file_path = os.path.join(test_logpath, constants.SideBySide.HTML_FILE)
9633+
out_file = codecs.open(file_path, "w+", encoding="utf-8")
9634+
out_file.writelines(the_html)
9635+
out_file.close()
9636+
95519637
def check_window(
95529638
self,
95539639
name="default",
@@ -9658,7 +9744,10 @@ def check_window(
96589744
baseline_dir = constants.VisualBaseline.STORAGE_FOLDER
96599745
visual_baseline_path = baseline_dir + "/" + test_id + "/" + name
96609746
page_url_file = visual_baseline_path + "/page_url.txt"
9661-
screenshot_file = visual_baseline_path + "/screenshot.png"
9747+
baseline_png = "baseline.png"
9748+
baseline_png_path = visual_baseline_path + "/%s" % baseline_png
9749+
latest_png = "latest.png"
9750+
latest_png_path = visual_baseline_path + "/%s" % latest_png
96629751
level_1_file = visual_baseline_path + "/tags_level_1.txt"
96639752
level_2_file = visual_baseline_path + "/tags_level_2.txt"
96649753
level_3_file = visual_baseline_path + "/tags_level_3.txt"
@@ -9674,7 +9763,7 @@ def check_window(
96749763
pass # Only reachable during multi-threaded test runs
96759764
if not os.path.exists(page_url_file):
96769765
set_baseline = True
9677-
if not os.path.exists(screenshot_file):
9766+
if not os.path.exists(baseline_png_path):
96789767
set_baseline = True
96799768
if not os.path.exists(level_1_file):
96809769
set_baseline = True
@@ -9694,7 +9783,9 @@ def check_window(
96949783
level_3 = json.loads(json.dumps(level_3)) # Tuples become lists
96959784

96969785
if set_baseline:
9697-
self.save_screenshot("screenshot.png", visual_baseline_path)
9786+
self.save_screenshot(
9787+
baseline_png, visual_baseline_path, selector="body"
9788+
)
96989789
out_file = codecs.open(page_url_file, "w+", encoding="utf-8")
96999790
out_file.writelines(page_url)
97009791
out_file.close()
@@ -9708,7 +9799,26 @@ def check_window(
97089799
out_file.writelines(json.dumps(level_3))
97099800
out_file.close()
97109801

9802+
test_logpath = os.path.join(self.log_path, self.__get_test_id())
9803+
baseline_path = os.path.join(visual_baseline_path, baseline_png)
9804+
baseline_copy_name = "baseline_%s.png" % name
9805+
baseline_copy_path = os.path.join(test_logpath, baseline_copy_name)
9806+
b_c_alt_name = "baseline.png"
9807+
b_c_alt_path = os.path.join(test_logpath, b_c_alt_name)
9808+
latest_copy_name = "baseline_diff_%s.png" % name
9809+
latest_copy_path = os.path.join(test_logpath, latest_copy_name)
9810+
l_c_alt_name = "baseline_diff.png"
9811+
l_c_alt_path = os.path.join(test_logpath, l_c_alt_name)
9812+
baseline_copy_tuple = (
9813+
baseline_path, baseline_copy_path, b_c_alt_path,
9814+
latest_png_path, latest_copy_path, l_c_alt_path,
9815+
)
9816+
self.__visual_baseline_copies.append(baseline_copy_tuple)
9817+
97119818
if not set_baseline:
9819+
self.save_screenshot(
9820+
latest_png, visual_baseline_path, selector="body"
9821+
)
97129822
f = open(page_url_file, "r")
97139823
page_url_data = f.read().strip()
97149824
f.close()
@@ -9805,6 +9915,8 @@ def check_window(
98059915
except Exception as e:
98069916
print(e) # Level-0 Dry Run (Only print the differences)
98079917
unittest.TestCase.maxDiff = None # Reset unittest.TestCase.maxDiff
9918+
# Since the check passed, do not save an extra copy of the baseline
9919+
del self.__visual_baseline_copies[-1] # .pop() returns the element
98089920

98099921
############
98109922

@@ -11579,14 +11691,18 @@ def tearDown(self):
1157911691
settings.LARGE_TIMEOUT = sb_config._LARGE_TIMEOUT
1158011692
sb_config._is_timeout_changed = False
1158111693
self.__overrided_default_timeouts = False
11694+
deferred_exception = None
1158211695
if self.__deferred_assert_failures:
1158311696
print(
1158411697
"\nWhen using self.deferred_assert_*() methods in your tests, "
1158511698
"remember to call self.process_deferred_asserts() afterwards. "
1158611699
"Now calling in tearDown()...\nFailures Detected:"
1158711700
)
1158811701
if not has_exception:
11589-
self.process_deferred_asserts()
11702+
try:
11703+
self.process_deferred_asserts()
11704+
except Exception as e:
11705+
deferred_exception = e
1159011706
else:
1159111707
self.process_deferred_asserts(print_only=True)
1159211708
if self.is_pytest:
@@ -11784,5 +11900,11 @@ def tearDown(self):
1178411900
self._last_page_url = self.get_current_url()
1178511901
except Exception:
1178611902
self._last_page_url = "(Error: Unknown URL)"
11787-
# Finally close all open browser windows
11903+
# (Nosetests) Finally close all open browser windows
1178811904
self.__quit_all_drivers()
11905+
# Resume tearDown() for both Pytest and Nosetests
11906+
if has_exception and self.__visual_baseline_copies:
11907+
self.__process_visual_baseline_logs()
11908+
if deferred_exception:
11909+
# User forgot to call "self.process_deferred_asserts()" in test
11910+
raise deferred_exception

‎seleniumbase/fixtures/constants.py

Lines changed: 7 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,8 @@
22
SeleniumBase constants are stored in this file.
33
"""
44

5+
from seleniumbase.core import encoded_images
6+
57

68
class Environment:
79
# Usage Example => "--env=qa" => Then access value in tests with "self.env"
@@ -32,8 +34,6 @@ class Recordings:
3234

3335

3436
class Dashboard:
35-
from seleniumbase.core import encoded_images
36-
3737
TITLE = "SeleniumBase Test Results Dashboard"
3838
# STYLE_CSS = "https://seleniumbase.io/cdn/css/pytest_style.css"
3939
STYLE_CSS = "assets/pytest_style.css" # Generated before tests
@@ -52,6 +52,11 @@ class Dashboard:
5252
DASH_PIE_PNG_3 = encoded_images.DASH_PIE_PNG_3 # Faster than CDN
5353

5454

55+
class SideBySide:
56+
HTML_FILE = "side_by_side.html"
57+
SIDE_BY_SIDE_PNG = encoded_images.SIDE_BY_SIDE_PNG
58+
59+
5560
class MultiBrowser:
5661
CHROMEDRIVER_FIXING_LOCK = Files.DOWNLOADS_FOLDER + "/driver_fixing.lock"
5762
CHROMEDRIVER_REPAIRED = Files.DOWNLOADS_FOLDER + "/driver_fixed.lock"

0 commit comments

Comments
 (0)