merge main

Author: ben-c-at-moz

.github/workflows/smoke.yml

@@ -0,0 +1,69 @@
+---
+name: Mac and Windows Smoke
+
+run-name: ${{ github.actor }} is running smoke tests
+on:
+  pull_request:
+
+jobs:
+  Smoke-Windows:
+    runs-on: windows-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          mkdir -p artifacts;
+          pip3 install 'pipenv==2023.11.15';
+          pip3 install 'ruff>=0.4.8,<0.5';
+          bash collect_executables.sh;
+          sleep 3;
+          rm ./pyproject.toml;
+          mv ./ci_pyproject.toml ./pyproject.toml;
+          pipenv install;
+      - name: Run Smoke Tests in Win
+        run: |
+          pipenv run pytest --fx-executable ./firefox/firefox -n 4 .
+          $env:TEST_EXIT_CODE = $LASTEXITCODE
+          mv artifacts artifacts-win || true
+          exit $env:TEST_EXIT_CODE
+      - name: Upload artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifacts-win
+          path: artifacts-win
+  Smoke-MacOS:
+    runs-on: macos-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Install dependencies
+        run: |
+          mkdir -p artifacts;
+          pip3 install 'pipenv==2023.11.15';
+          pip3 install 'ruff>=0.4.8,<0.5';
+          bash collect_executables.sh;
+          sleep 3;
+          rm ./pyproject.toml;
+          mv ./ci_pyproject.toml ./pyproject.toml;
+          pipenv install;
+      - name: Run Smoke Tests in MacOS
+        run: |
+          pipenv run pytest --fx-executable ./firefox/firefox -s -n 4 . || TEST_EXIT_CODE=$?
+          mv artifacts artifacts-mac || true
+          exit $TEST_EXIT_CODE
+      - name: Upload artifacts
+        if: always()
+        uses: actions/upload-artifact@v4
+        with:
+          name: artifacts-mac
+          path: artifacts-mac

.gitignore

@@ -2,6 +2,8 @@
 results.txt
 err_log.txt
 **/report.html
+**/result.xml
+**/report.xml
 assets/
 html_dump/
 /Test Results - .html
@@ -13,6 +15,7 @@ env
 *.zip
 *.gz
 *.7z
+*.xml
 test.sh
 file[0-9]*.txt
 

.taskcluster.yml

@@ -183,7 +183,6 @@ tasks:
                                     ${normProjectUpper}_HEAD_REF: '${short_head_ref}'
                                     ${normProjectUpper}_HEAD_REV: '${head_sha}'
                                     ${normProjectUpper}_REPOSITORY_TYPE: git
-                                    ${normProjectUpper}_PIP_REQUIREMENTS: taskcluster/requirements.txt
                                     REPOSITORIES:
                                         $json:
                                             ${normProject}: ${normProject}
@@ -203,7 +202,7 @@ tasks:
                           features:
                               taskclusterProxy: true
 
-                          image: mozillareleases/taskgraph:decision-c4ac262880970ca484105929e02dd12f00214d5f2603ab6ce4c0d17de5cd0280@sha256:2dd667994aa13fccdcdcede85c570a5eb4a5247e42875a9e95a135ef774ee469
+                          image: mozillareleases/taskgraph:decision-v9.0.0@sha256:e56c7e5cd467c2ce497c344b358f68cc84a4f73f3422e507a97b397d4e617fbd
                           maxRunTime: 1800
 
                           command:
@@ -220,11 +219,11 @@ tasks:
                                         cd /builds/worker/checkouts/src &&
                                         ln -s /builds/worker/artifacts artifacts &&
                                         pip3 install -r requirements/base.txt &&
-                                        ~/.local/bin/taskgraph action-callback
+                                        taskgraph action-callback
                                     else: >
                                         cd /builds/worker/checkouts/src &&
                                         ln -s /builds/worker/artifacts artifacts &&
-                                        ~/.local/bin/taskgraph decision
+                                        taskgraph decision
                                         --pushlog-id='0'
                                         --pushdate='0'
                                         --project='${project}'

Pipfile

@@ -13,14 +13,15 @@ requests = "*"
 pytest-xdist = "*"
 pytest-html = "*"
 pypom = "*"
+taskcluster-taskgraph = "==9.0.0"
 jsonpath-ng = "*"
 pillow = "*"
 pyfxa = "*"
+ruff = "0.5.2"
+pytest-rerunfailures = "*"
 
 [dev-packages]
 werkzeug = "*"
 selenium-wire = "*"
 pdoc = "*"
-ruff = "0.4.3"
 pytest-variables = "*"
-taskcluster-taskgraph = "*"

README.md

@@ -1,4 +1,6 @@
-## fx-desktop-qa-automation
+## Project STARfox
+**S**moke **T**est **A**utomation **R**epository for Fire*fox* Desktop
+
 A Python Selenium based set of tests for smoke testing of Firefox, including Fx incident smoke testing.
 
 ### Build under test

ci_pyproject.toml

@@ -16,7 +16,7 @@ markers = [
 testpaths = [
     "tests"
 ]
-addopts = "-vs --ci -m 'not incident and not unstable' --html=artifacts/report.html"
+addopts = "-vs --ci --reruns 2 --reruns-delay 1 -m 'not incident and not unstable' --html=artifacts/report.html"
 
 [tool.ruff]
 target-version = "py310"

collect_executables.sh

@@ -1,13 +1,22 @@
 #!/bin/bash
+echo "~~collect executables~~"
 
 # Usage: ./collect_executables.sh [channel]
 # Collects geckodriver and Fx, default channel is Beta.
 
 ## Determine OS and arch
 UNAME_A=$(uname -a)
-if [ -n "$WSL_DISTRO_NAME" ]
+## Save the system arch info
+echo "uname -a: ${UNAME_A}"
+if [ -n "$WSL_DISTRO_NAME" ] || [[ $UNAME_A == *"MINGW"* ]]
 then
     SYSTEM_NAME="win"
+    if [[ $UNAME_A == *"x86_64"* ]]
+    then
+        BITS="64"
+    else
+        BITS="32"
+    fi
 else
     if [[ $UNAME_A == *"Darwin"* ]]
     then
@@ -29,11 +38,6 @@ else
     fi
 fi
 
-if [[ "$SYSTEM_NAME" == "win" ]] && [[ -z $ARCH ]] && [[ $BITS = "64" ]]
-then
-    BITS=32
-fi
-
 if [[ $SYSTEM_NAME == "win" ]]
 then
     EXT="zip"
@@ -43,14 +47,13 @@ fi
 
 # Find the version of Geckodriver that matches arch
 FILENAME="-${SYSTEM_NAME}${BITS}${ARCH}.${EXT}"
-
+echo "FILENAME ${FILENAME}"
 # 20 is arbitrary and may break if future releases of Geckodriver have more than 20 channels
 for i in {0..20}
 do
     GECKO_LINK=$(curl -s https://api.github.com/repos/mozilla/geckodriver/releases/latest | jq ".[\"assets\"][${i}][\"browser_download_url\"]" | tr -d '"')
     if [[ $GECKO_LINK == *"${FILENAME}"* ]] && [[ $GECKO_LINK != *".asc" ]]
     then
-        echo "$GECKO_LINK"
         curl -OL "$GECKO_LINK"
     fi
 done
@@ -83,9 +86,26 @@ FX_LOC=$(echo "$FX_LINK_HTML" | awk -F '"' '{print $2}')
 
 curl -O "$FX_LOC"
 
-mv geckodriver*.tar.gz geckodriver.tar.gz
-tar -xvzf geckodriver.tar.gz
+GD_FILE=$(ls geckodriver*)
+mv "$GD_FILE" "geckodriver.${EXT}"
+if [[ $EXT == "zip" ]]
+then
+    unzip geckodriver.zip
+else
+    tar -xvzf geckodriver.tar.gz
+fi
+
+# Wait up to 10 seconds for geckodriver to exist
+for ((i=0; i<200; i++))
+do
+    if [ -f geckodriver ]
+    then
+        break
+    fi
+    sleep 0.2
+done
 chmod +x geckodriver
+./geckodriver --version
 
 if [[ $SYSTEM_NAME == "linux" ]]
 then

conftest.py

@@ -1,11 +1,75 @@
+import datetime
 import logging
 import os
 import platform
+import re
 from typing import Callable, List, Tuple
 
 import pytest
-from selenium import webdriver
+from selenium.common.exceptions import TimeoutException, WebDriverException
+from selenium.webdriver import Firefox
+from selenium.webdriver.common.by import By
 from selenium.webdriver.firefox.options import Options
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+
+def screenshot_content(driver: Firefox, opt_ci: bool, test_name: str) -> None:
+    """
+    Screenshots the current browser, saves with appropriate test name and date for reference
+    """
+    current_time = str(datetime.datetime.now())
+    current_time = re.sub(r"[^\w_. -]", "_", current_time)
+    filename = f"{test_name}_{current_time}_image"
+    _screenshot(filename, driver, opt_ci)
+
+
+def log_content(opt_ci: bool, driver: Firefox, test_name: str) -> None:
+    """
+    Logs the current browser content, with the appropriate test name and date for reference.
+    """
+    artifacts_loc = "artifacts" if opt_ci else ""
+    current_time = str(datetime.datetime.now())
+    current_time = re.sub(r"[^\w_. -]", "_", current_time)
+    fullpath_chrome = os.path.join(
+        artifacts_loc, f"{test_name}_{current_time}_content.txt"
+    )
+    fullpath_content = os.path.join(
+        artifacts_loc, f"{test_name}_{current_time}_chrome.txt"
+    )
+
+    # Save Chrome context page source
+    with open(fullpath_chrome, "w", encoding="utf-8") as fh:
+        with driver.context(driver.CONTEXT_CHROME):
+            output_contents = driver.page_source
+            fh.write(output_contents)
+
+    # Save Content context page source
+    with open(fullpath_content, "w", encoding="utf-8") as fh:
+        output_contents = driver.page_source
+        fh.write(output_contents)
+    return
+
+
+def pytest_exception_interact(node, call, report):
+    """
+    Method that wraps all test execution, on any exception/failure an artifact with the information about the failure is kept.
+    """
+    if report.failed:
+        try:
+            test_name = node.name
+            logging.error(f"Handling exception for test: {test_name}")
+            logging.error(
+                f"NODE LOGS HERE {node.funcargs}\n THE FAILED TEST: {test_name}"
+            )
+            driver = node.funcargs.get("driver")
+            opt_ci = node.funcargs.get("opt_ci")
+            if driver:
+                log_content(opt_ci, driver, test_name)
+                screenshot_content(driver, opt_ci, test_name)
+        except Exception as e:
+            logging.warning("Something went wrong with the exception catching.")
+            raise e
 
 
 def pytest_addoption(parser):
@@ -53,6 +117,17 @@ def pytest_addoption(parser):
     )
 
 
+def _screenshot(filename: str, driver: Firefox, opt_ci: bool):
+    if not filename.endswith(".png"):
+        filename = filename + ".png"
+    artifacts_loc = ""
+    if opt_ci:
+        artifacts_loc = "artifacts"
+    fullpath = os.path.join(artifacts_loc, filename)
+    driver.save_screenshot(fullpath)
+    return fullpath
+
+
 @pytest.fixture()
 def opt_headless(request):
     return request.config.getoption("--run-headless")
@@ -166,7 +241,7 @@ def driver(
         options.binary_location = fx_executable
         for opt, value in set_prefs:
             options.set_preference(opt, value)
-        driver = webdriver.Firefox(options=options)
+        driver = Firefox(options=options)
         separator = "x"
         if separator not in opt_window_size:
             if "by" in opt_window_size:
@@ -179,36 +254,31 @@ def driver(
         driver.set_window_size(*winsize)
         timeout = 30 if opt_ci else opt_implicit_timeout
         driver.implicitly_wait(timeout)
+        WebDriverWait(driver, timeout=40).until(
+            EC.presence_of_element_located((By.TAG_NAME, "body"))
+        )
         yield driver
-
+    except (WebDriverException, TimeoutException) as e:
+        logging.warning(f"DRIVER exception: {e}")
     finally:
-        driver.quit()
+        if "driver" in locals() or "driver" in globals():
+            driver.quit()
 
 
 @pytest.fixture()
-def screenshot(driver: webdriver.Firefox, opt_ci: bool) -> Callable:
+def screenshot(driver: Firefox, opt_ci: bool) -> Callable:
     """
     Factory fixture that returns a screenshot function.
     """
 
-    def _screenshot(filename: str) -> str:
-        """
-        Given a short filename, save a screenshot and return the image's full path.
-        """
-        if not filename.endswith(".png"):
-            filename = filename + ".png"
-        artifacts_loc = ""
-        if opt_ci:
-            artifacts_loc = "artifacts"
-        fullpath = os.path.join(artifacts_loc, filename)
-        driver.save_screenshot(fullpath)
-        return fullpath
+    def screenshot_wrapper(filename: str) -> str:
+        return _screenshot(filename, driver, opt_ci)
 
-    return _screenshot
+    return screenshot_wrapper
 
 
 @pytest.fixture()
-def version(driver: webdriver.Firefox):
+def version(driver: Firefox):
     return driver.capabilities["browserVersion"]