Merge pull request #143 from mozilla/sl/exception-wrapper

Exception Wrapper

Author: sarinali

.github/workflows/smoke.yml

@@ -28,8 +28,11 @@ jobs:
       - name: Run Smoke Tests in Win
         run: |
           pipenv run pytest --fx-executable ./firefox/firefox -n 4 .
-          mv artifacts artifacts-win
+          $env:TEST_EXIT_CODE = $LASTEXITCODE
+          mv artifacts artifacts-win || true
+          exit $env:TEST_EXIT_CODE
       - name: Upload artifacts
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: artifacts-win
@@ -55,9 +58,11 @@ jobs:
           pipenv install;
       - name: Run Smoke Tests in MacOS
         run: |
-          pipenv run pytest --fx-executable ./firefox/firefox -n 4 tests
-          mv artifacts artifacts-mac
+          pipenv run pytest --fx-executable ./firefox/firefox -s -n 4 . || TEST_EXIT_CODE=$?
+          mv artifacts artifacts-mac || true
+          exit $TEST_EXIT_CODE
       - name: Upload artifacts
+        if: always()
         uses: actions/upload-artifact@v4
         with:
           name: artifacts-mac

conftest.py

@@ -1,14 +1,75 @@
+import datetime
 import logging
 import os
 import platform
+import re
 from typing import Callable, List, Tuple
 
 import pytest
 from selenium import webdriver
 from selenium.common.exceptions import WebDriverException
+from selenium.webdriver import Firefox
 from selenium.webdriver.firefox.options import Options
 
 
+def screenshot_content(driver: Firefox, opt_ci: bool, test_name: str) -> None:
+    """
+    Screenshots the current browser, saves with appropriate test name and date for reference
+    """
+    current_time = str(datetime.datetime.now())
+    current_time = re.sub(r"[^\w_. -]", "_", current_time)
+    filename = f"{test_name}_{current_time}_image"
+    _screenshot(filename, driver, opt_ci)
+
+
+def log_content(opt_ci: bool, driver: Firefox, test_name: str) -> None:
+    """
+    Logs the current browser content, with the appropriate test name and date for reference.
+    """
+    artifacts_loc = "artifacts" if opt_ci else ""
+    current_time = str(datetime.datetime.now())
+    current_time = re.sub(r"[^\w_. -]", "_", current_time)
+    fullpath_chrome = os.path.join(
+        artifacts_loc, f"{test_name}_{current_time}_content.txt"
+    )
+    fullpath_content = os.path.join(
+        artifacts_loc, f"{test_name}_{current_time}_chrome.txt"
+    )
+
+    # Save Chrome context page source
+    with open(fullpath_chrome, "w", encoding="utf-8") as fh:
+        with driver.context(driver.CONTEXT_CHROME):
+            output_contents = driver.page_source
+            fh.write(output_contents)
+
+    # Save Content context page source
+    with open(fullpath_content, "w", encoding="utf-8") as fh:
+        output_contents = driver.page_source
+        fh.write(output_contents)
+    return
+
+
+def pytest_exception_interact(node, call, report):
+    """
+    Method that wraps all test execution, on any exception/failure an artifact with the information about the failure is kept.
+    """
+    if report.failed:
+        try:
+            test_name = node.name
+            logging.error(f"Handling exception for test: {test_name}")
+            logging.error(
+                f"NODE LOGS HERE {node.funcargs}\n THE FAILED TEST: {test_name}"
+            )
+            driver = node.funcargs.get("driver")
+            opt_ci = node.funcargs.get("opt_ci")
+            if driver:
+                log_content(opt_ci, driver, test_name)
+                screenshot_content(driver, opt_ci, test_name)
+        except Exception as e:
+            logging.warning("Something went wrong with the exception catching.")
+            raise e
+
+
 def pytest_addoption(parser):
     """Set custom command-line options"""
     parser.addoption(
@@ -54,6 +115,17 @@ def pytest_addoption(parser):
     )
 
 
+def _screenshot(filename: str, driver: Firefox, opt_ci: bool):
+    if not filename.endswith(".png"):
+        filename = filename + ".png"
+    artifacts_loc = ""
+    if opt_ci:
+        artifacts_loc = "artifacts"
+    fullpath = os.path.join(artifacts_loc, filename)
+    driver.save_screenshot(fullpath)
+    return fullpath
+
+
 @pytest.fixture()
 def opt_headless(request):
     return request.config.getoption("--run-headless")
@@ -194,20 +266,10 @@ def screenshot(driver: webdriver.Firefox, opt_ci: bool) -> Callable:
     Factory fixture that returns a screenshot function.
     """
 
-    def _screenshot(filename: str) -> str:
-        """
-        Given a short filename, save a screenshot and return the image's full path.
-        """
-        if not filename.endswith(".png"):
-            filename = filename + ".png"
-        artifacts_loc = ""
-        if opt_ci:
-            artifacts_loc = "artifacts"
-        fullpath = os.path.join(artifacts_loc, filename)
-        driver.save_screenshot(fullpath)
-        return fullpath
-
-    return _screenshot
+    def screenshot_wrapper(filename: str) -> str:
+        return _screenshot(filename, driver, opt_ci)
+
+    return screenshot_wrapper
 
 
 @pytest.fixture()