From bc30e3aa0873fc020cc8b2dff0b431641d4ad271 Mon Sep 17 00:00:00 2001 From: James Graham Date: Thu, 7 Aug 2025 11:01:07 +0000 Subject: [PATCH 1/2] Allow wpt reftest to avoid caching screenshots This is helpful if we have a stateful issue where e.g. font rendering may change between tests, so a test compared with a screenshot of the same reference taken for an earlier test might fail. Clearly in such a situation we should fix the underlying bug, but disabling caches is preferable to mistakenly classifying many tests as failing. In addition, disable screenshot caches for Android 14, since that platform seems to have such an issue. Differential Revision: https://phabricator.services.mozilla.com/D260245 bugzilla-url: https://bugzilla.mozilla.org/show_bug.cgi?id=1981664 gecko-commit: 40ef569ca4d81bdbc49933c704b3f0a6076bb1eb gecko-reviewers: Sasha, webdriver-reviewers --- tools/wptrunner/wptrunner/browsers/firefox.py | 9 +++++++++ .../wptrunner/executors/executormarionette.py | 12 +++++++++--- 2 files changed, 18 insertions(+), 3 deletions(-) diff --git a/tools/wptrunner/wptrunner/browsers/firefox.py b/tools/wptrunner/wptrunner/browsers/firefox.py index 2324e043f30bbd..61ca6aa42c0a9e 100644 --- a/tools/wptrunner/wptrunner/browsers/firefox.py +++ b/tools/wptrunner/wptrunner/browsers/firefox.py @@ -164,6 +164,15 @@ def executor_kwargs(logger, test_type, test_environment, run_info_data, capabilities["pageLoadStrategy"] = "eager" if test_type in ("reftest", "print-reftest"): executor_kwargs["reftest_internal"] = kwargs["reftest_internal"] + cache_screenshots = True + if run_info_data["os"] == "android": + try: + major_version = int(run_info_data["version"].split(".", 1)[0]) + except ValueError: + pass + else: + cache_screenshots = major_version < 14 + executor_kwargs["cache_screenshots"] = cache_screenshots if test_type == "wdspec": options = {"args": []} if kwargs["binary"]: diff --git a/tools/wptrunner/wptrunner/executors/executormarionette.py b/tools/wptrunner/wptrunner/executors/executormarionette.py index 3607960b41e513..37c64e1745b3de 100644 --- a/tools/wptrunner/wptrunner/executors/executormarionette.py +++ b/tools/wptrunner/wptrunner/executors/executormarionette.py @@ -1059,7 +1059,8 @@ def __init__(self, logger, browser, server_config, timeout_multiplier=1, debug_info=None, reftest_internal=False, reftest_screenshot="unexpected", ccov=False, group_metadata=None, capabilities=None, debug=False, - browser_version=None, debug_test=False, **kwargs): + browser_version=None, debug_test=False, + cache_screenshots=True, **kwargs): """Marionette-based executor for reftests""" RefTestExecutor.__init__(self, logger, @@ -1082,6 +1083,7 @@ def __init__(self, logger, browser, server_config, timeout_multiplier=1, self.group_metadata = group_metadata self.debug = debug self.debug_test = debug_test + self.cache_screenshots = cache_screenshots self.install_extensions = browser.extensions @@ -1225,7 +1227,9 @@ def logger(self): return self.executor.logger def setup(self, screenshot="unexpected", chrome_scope=False): - data = {"screenshot": screenshot, "isPrint": self.executor.is_print} + data = {"screenshot": screenshot, + "isPrint": self.executor.is_print, + "cacheScreenshots": self.executor.cache_screenshots} if self.executor.group_metadata is not None: data["urlCount"] = {urljoin(self.executor.server_url(key[0]), key[1]):value for key, value in self.executor.group_metadata.get("url_count", {}).items() @@ -1234,6 +1238,7 @@ def setup(self, screenshot="unexpected", chrome_scope=False): if chrome_scope: self.logger.debug("Using marionette Chrome scope for reftests") self.executor.protocol.marionette.set_context(self.executor.protocol.marionette.CONTEXT_CHROME) + self.logger.debug(f"Starting internal reftests with {data}") self.executor.protocol.marionette._send_message("reftest:setup", data) def reset(self, **kwargs): @@ -1357,7 +1362,7 @@ def __init__(self, logger, browser, server_config, timeout_multiplier=1, screenshot_cache=None, close_after_done=True, debug_info=None, reftest_screenshot="unexpected", ccov=False, group_metadata=None, capabilities=None, debug=False, - reftest_internal=False, **kwargs): + reftest_internal=False, cache_screenshots=True, **kwargs): """Marionette-based executor for reftests""" MarionetteRefTestExecutor.__init__(self, logger, @@ -1373,6 +1378,7 @@ def __init__(self, logger, browser, server_config, timeout_multiplier=1, group_metadata=group_metadata, capabilities=capabilities, debug=debug, + cache_screenshots=cache_screenshots, **kwargs) def setup(self, runner, protocol=None): From 6549603b14925317894ffd5ff90d2f380651f280 Mon Sep 17 00:00:00 2001 From: Alexandra Borovova Date: Fri, 8 Aug 2025 09:57:29 +0200 Subject: [PATCH 2/2] Fix linting error --- tools/wptrunner/wptrunner/browsers/firefox.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/wptrunner/wptrunner/browsers/firefox.py b/tools/wptrunner/wptrunner/browsers/firefox.py index 61ca6aa42c0a9e..bdabeb46f1eed5 100644 --- a/tools/wptrunner/wptrunner/browsers/firefox.py +++ b/tools/wptrunner/wptrunner/browsers/firefox.py @@ -167,7 +167,7 @@ def executor_kwargs(logger, test_type, test_environment, run_info_data, cache_screenshots = True if run_info_data["os"] == "android": try: - major_version = int(run_info_data["version"].split(".", 1)[0]) + major_version = int(run_info_data["version"].split(".", 1)[0]) except ValueError: pass else: