Merge branch 'main' into Hani/testrail_set_all_subs_to_functional

Author: Hani Yacoub

.github/workflows/smoke.yml

@@ -1,5 +1,5 @@
 ---
-name: Mac and Windows Smoke
+name: Smoke Test Execution
 
 run-name: ${{ github.actor }} is running smoke tests
 on:
@@ -14,6 +14,20 @@ on:
       job_to_run:
         required: true
         type: string
+  workflow_dispatch:
+    inputs:
+      win_installer_link:
+        description: 'The link to the Windows installer (win64) for the Fx under test'
+        required: false
+        type: string
+      mac_installer_link:
+        description: 'The link to the macOS installer for the Fx under test'
+        required: false
+        type: string
+      linux_tarball_link:
+        description: 'The link to the Linux tarball (linux-x86_64) for the Fx under test'
+        required: false
+        type: string
 env:
   FX_CHANNEL: ${{ inputs.channel }}
   TESTRAIL_BASE_URL: ${{ secrets.TESTRAIL_BASE_URL }}
@@ -23,7 +37,7 @@ env:
 
 jobs:
   Smoke-Windows:
-    if: ${{ inputs.job_to_run == 'Smoke-Windows' || github.event_name == 'pull_request' }}
+    if: ${{ inputs.job_to_run == 'Smoke-Windows' || github.event_name == 'pull_request' || inputs.win_installer_link }}
     runs-on: windows-latest
     steps:
       - name: Create app token
@@ -40,7 +54,15 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
-      - name: Set Environment
+      - name: Set Environment (Manual)
+        if: ${{ inputs.win_installer_link }}
+        run: |
+          $env_contents = @"
+          MANUAL='true'
+          "@
+          New-item -Name .env -Value $env_contents -ItemType File -Force
+          Write-Host "Running smoke tests on supplied executable"
+      - name: Set Environment (Scheduled Beta)
         if: ${{ inputs.job_to_run == 'Smoke-Windows' }}
         run: |
           $env_contents = @"
@@ -65,12 +87,19 @@ jobs:
           unzip geckodriver.zip
         shell: pwsh
       - name: Download Beta
+        if: ${{ !inputs.win_installer_link }}
         run: |
           $fx_url = (pipenv run python collect_executables.py) -join "`n"
           echo $fx_url
           Invoke-WebRequest -Uri $fx_url -OutFile "${{ github.workspace }}\setup.exe"
           New-Item -ItemType Directory -Path "C:\Program Files\Custom Firefox" -Force
         shell: pwsh
+      - name: Download Executable
+        if: ${{ inputs.win_installer_link }}
+        run: |
+          Invoke-WebRequest -Uri ${{ inputs.win_installer_link }} -OutFile "${{ github.workspace }}\setup.exe"
+          New-Item -ItemType Directory -Path "C:\Program Files\Custom Firefox" -Force
+        shell: pwsh
       - name: Install Beta
         id: setup
         run: |
@@ -117,7 +146,7 @@ jobs:
           name: artifacts-win
           path: artifacts-win
   Smoke-MacOS:
-    if: ${{ inputs.job_to_run == 'Smoke-MacOS' || github.event_name == 'pull_request' }}
+    if: ${{ inputs.job_to_run == 'Smoke-MacOS' || github.event_name == 'pull_request' || inputs.mac_installer_link }}
     runs-on: macos-latest
     steps:
       - name: Create app token
@@ -139,20 +168,29 @@ jobs:
           python check_test_cases.py
           exit $?
           echo "Triggered by event: ${{ github.event_name }}"
-      - name: Set Environment
+      - name: Set Environment (Scheduled Beta)
         if: ${{ inputs.job_to_run == 'Smoke-MacOS' }}
         run: |
           echo "TESTRAIL_REPORT='true'" >> "$GITHUB_ENV";
           echo "Running report for most recent Beta on MacOS";
+      - name: Set Environment (Manual)
+        if: ${{ inputs.mac_installer_link }}
+        run: |
+          echo "MANUAL='true'" >> "$GITHUB_ENV";
+          echo "Running smoke tests on supplied executable";
       - name: Install dependencies
-        id: setup
         run: |
           mkdir -p artifacts;
           pip3 install 'pipenv==2023.11.15';
           pip3 install 'ruff>=0.4.8,<0.5';
           rm ./pyproject.toml;
           mv ./ci_pyproject.toml ./pyproject.toml;
-          pipenv install;
+          pipenv install
+      - name: Install Fx
+        id: setup
+        env:
+          MANUAL_DOWNLOAD_LINK: ${{ inputs.mac_installer_link }}
+        run:
           echo app_name=$(bash ./collect_executables.sh | xargs -0 ./parse_executables.sh) >> "$GITHUB_OUTPUT"
       - name: Run Smoke Tests in MacOS
         if: steps.setup.conclusion == 'success'
@@ -181,8 +219,69 @@ jobs:
         with:
           name: artifacts-mac
           path: artifacts-mac
+  Smoke-Linux:
+    if: ${{ inputs.linux_tarball_link }}
+    runs-on: ubuntu-latest
+    steps:
+      - name: Create app token
+        uses: actions/create-github-app-token@v1
+        id: app-token
+        with:
+          app-id: ${{ secrets.BOT_CLIENT_ID }}
+          private-key: ${{ secrets.BOT_PRIVATE_KEY }}
+      - name: Checkout repository
+        uses: actions/checkout@v4
+        with:
+          token: ${{ steps.app-token.outputs.token }}
+      - name: Set up Python 3.11
+        uses: actions/setup-python@v5
+        with:
+          python-version: "3.11"
+      - name: Check test case numbers
+        run: |
+          python check_test_cases.py
+          exit $?
+          echo "Triggered by event: ${{ github.event_name }}"
+      - name: Install dependencies
+        id: setup
+        env:
+          MANUAL_DOWNLOAD_LINK: ${{ inputs.linux_tarball_link }}
+        run: |
+          echo "MANUAL='true'" >> "$GITHUB_ENV";
+          echo "Running smoke tests on supplied executable";
+          sudo apt install gnome-screenshot
+          uname -m;
+          mkdir -p artifacts;
+          pip3 install 'pipenv==2023.11.15';
+          pip3 install 'ruff>=0.4.8,<0.5';
+          rm ./pyproject.toml;
+          mv ./ci_xvfb_pyproject.toml ./pyproject.toml;
+          pipenv install;
+          ./collect_executables.sh
+          ./firefox/firefox --version;
+          #. ./keyring-unlock.sh;
+      - name: Run Smoke Tests in Ubuntu
+        if: steps.setup.conclusion == 'success'
+        env:
+          FX_EXECUTABLE: ./firefox/firefox
+        run: |
+          "$FX_EXECUTABLE" --version;
+          pipenv run python choose_ci_set.py;
+          Xvfb :99 -screen 0 '1600x1200x24' > artifacts/xvfb.log &
+          DISPLAY=:99 pipenv run pytest --fx-executable="$FX_EXECUTABLE" -n 4 $(cat selected_tests) || TEST_EXIT_CODE=$?
+          exit $TEST_EXIT_CODE
+      - name: Run Smoke Tests in Ubuntu (Headed)
+        if: steps.setup.conclusion == 'success' && always()
+        env:
+          FX_EXECUTABLE: ./firefox/firefox
+        run: |
+          mv ./ci_xvfb_pyproject_headed.toml ./pyproject.toml;
+          pipenv run python choose_ci_set.py
+          DISPLAY=:99 pipenv run pytest --fx-executable="$FX_EXECUTABLE" $(cat selected_tests) || TEST_EXIT_CODE=$?
+          exit $TEST_EXIT_CODE
 
   Use-Artifacts:
+    if: ${{ github.event_name != 'workflow_dispatch' }}
     runs-on: ubuntu-latest
     needs:
       - Smoke-Windows

.gitignore

@@ -194,6 +194,7 @@ tests/test_scratch.py
 
 # Credentials
 credentials.json
+testrail_credentials.env
 
 # Do not ignore
 !data/goomy.png

README.md

@@ -127,6 +127,28 @@ You may find that if you are re-running all previously executed test runs that y
 "Session is not reportable." If you wish to overwrite a previously reported session, add
 `REPORTABLE=true` to your environment.
 
+### Manual Execution of Smoke Tests
+
+To run the smoke tests manually against an arbitrary version of Firefox **where the installer or (for Linux)
+tarball exists on the web**:
+
+- Get write permissions to the repo (ask Ben C, Tracy, or Virgil S to add you)
+- Go to the [Smoke Test Execution workflow page](https://github.com/mozilla/fx-desktop-qa-automation/actions/workflows/smoke.yml)
+- At the top right of the workflow history panel (near the top right of the page), press the button
+  labeled "Run workflow"
+- Make sure you are running against `Branch: main`
+  - Alternately, runs can be made against ESR versions 128 and higher by running against `Branch: 00esr-manual`
+  - MacOS ESR manual runs are currently not supported
+- Add the URLs to your installers for Windows and MacOS, and the tarball for Linux in the fields below.
+  - **Installers and tarballs must be available on the internet**
+  - **Windows should be x86, 64-bit (win64), .exe installer (no .msi)**
+  - **MacOS should be a .dmg installer (no .pkg)**
+  - **Linux should be a .tar.xz or .tar.bz2 tarball (no .deb)**
+  - **Version numbers less than 128 are not likely to work**
+- Press the green "Run workflow" option
+- In a few minutes, the top workflow should be yours. Click through to watch progress.
+- Linux runs throw a lot of warnings, ignore them and focus on the results. Green checks are the key.
+
 ### Project Contribution
 If you are outside of Mozilla and would like to contribute to this project:
 - Read all the documentation including this README, BEST_PRACTICES and CODE_OF_CONDUCT.

SELECTOR_INFO.md

@@ -571,13 +571,6 @@ Location: about:preferences#privacy popup dialog
 Path to .json: modules/data/about_prefs.components.json
 ```
 ```
-Selector Name: dialog-close-button
-Selector Data: "dialogClose"
-Description: The close button of the popup dialog
-Location: top right corner of about:preferences popup dialog
-Path to .json: modules/data/about_prefs.components.json
-```
-```
 Selector Name: saved-addresses
 Selector Data: "addresses"
 Description: The element that contains the saved addresses
@@ -1368,14 +1361,7 @@ Location: Inside any autofill-eligible form field, triggered by user interaction
 Path to .json: modules/data/autofill_popup.components.json
 ```
 ```
-Selector Name: address-preview-form-container
-Selector Data: "address-save-update-notification-content"
-Description: Form container that is hidden in chrome context. Used to verify hover preview data
-Location: Inside chrome context, holds the data that would be showed as a preview when autofill is hovered.
-Path to .json: modules/data/autofill_popup.components.json
-```
-```
-Selector Name: cc-preview-form-container
+Selector Name: preview-form-container
 Selector Data: ".autocomplete-richlistbox richlistitem"
 Description: Form container that is hidden in chrome context. Used to verify hover preview data
 Location: Inside chrome context, holds the data that would be showed as a preview when autofill is hovered.
@@ -2630,7 +2616,7 @@ Path to .json: modules/data/navigation.components.json
 ```
 ```
 Selector name: sponsored-suggestion
-Selector Data: urlbarView-row-body-description
+Selector Data: urlbarView-group-aria-label
 Description: Sponsored search results
 Location: URL bar search results
 Path to .json: modules/data/navigation.components.json
@@ -2643,6 +2629,13 @@ Location: URL bar search results
 Path to .json: modules/data/navigation.components.json
 ```
 ```
+Selector name: suggestion-titles
+Selector Data: urlbarView-title
+Description: Firefox suggestion search results item titles
+Location: URL bar suggestion results
+Path to .json: modules/data/navigation.components.json
+```
+```
 Selector name: search-result-autofill-adaptive-element
 Selector Data: .//*[@type='autofill_adaptive']
 Description: Search result autofill adaptive element

choose_ci_set.py

@@ -107,11 +107,14 @@ def dedupe(run_list: list, slash: str) -> list:
 if __name__ == "__main__":
     if os.path.exists(".env"):
         with open(".env") as fh:
-            if "TESTRAIL_REPORT='true'" in fh.read():
+            contents = fh.read()
+            if "TESTRAIL_REPORT='true'" in contents:
                 os.environ["TESTRAIL_REPORT"] = "true"
+            if "RUN_ALL='true'" in contents:
+                os.environ["MANUAL"] = "true"
 
-    if os.environ.get("TESTRAIL_REPORT"):
-        # Run all tests if this is a scheduled beta
+    if os.environ.get("TESTRAIL_REPORT") or os.environ.get("MANUAL"):
+        # Run all tests if this is a scheduled beta or a manual run
         with open(OUTPUT_FILE, "w") as fh:
             fh.write("tests")
             sys.exit(0)

ci_pyproject.toml

@@ -11,7 +11,8 @@ markers = [
     "ci: basic tests to run in ci",
     "locale_de: tests run in DE locale versions",
     "locale_fr: tests run in FR locale versions",
-    "locale_gb: tests run in GB locale versions"
+    "locale_gb: tests run in GB locale versions",
+    "noxvfb: tests that should not run in xvfb sessions"
 ]
 testpaths = [
     "tests"

ci_pyproject_headed.toml

@@ -11,7 +11,8 @@ markers = [
     "ci: basic tests to run in ci",
     "locale_de: tests run in DE locale versions",
     "locale_fr: tests run in FR locale versions",
-    "locale_gb: tests run in GB locale versions"
+    "locale_gb: tests run in GB locale versions",
+    "noxvfb: tests that should not run in xvfb sessions"
 ]
 testpaths = [
     "tests"

ci_xvfb_pyproject.toml

@@ -0,0 +1,23 @@
+[tool.pytest.ini_options]
+generate_report_on_test = true
+log_cli = true
+log_cli_level = "warn"
+markers = [
+    "audio: test is reliant on audio",
+    "headed: test must run in headed mode (e.g. uses pynput)",
+    "incident: incident smoke tests",
+    "unstable: temporary mark for unstable tests",
+    "slow: test is clocked at more than 30s on modern machines",
+    "ci: basic tests to run in ci",
+    "locale_de: tests run in DE locale versions",
+    "locale_fr: tests run in FR locale versions",
+    "locale_gb: tests run in GB locale versions",
+    "noxvfb: tests that should not run in xvfb sessions"
+]
+testpaths = [
+    "tests"
+]
+addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json --reruns 3 --reruns-delay 3 -m 'not unstable and not headed and not noxvfb' --html=artifacts/report.html"
+
+[tool.ruff]
+target-version = "py310"

ci_xvfb_pyproject_headed.toml

@@ -0,0 +1,23 @@
+[tool.pytest.ini_options]
+generate_report_on_test = true
+log_cli = true
+log_cli_level = "warn"
+markers = [
+    "audio: test is reliant on audio",
+    "headed: test must run in headed mode (e.g. uses pynput)",
+    "incident: incident smoke tests",
+    "unstable: temporary mark for unstable tests",
+    "slow: test is clocked at more than 30s on modern machines",
+    "ci: basic tests to run in ci",
+    "locale_de: tests run in DE locale versions",
+    "locale_fr: tests run in FR locale versions",
+    "locale_gb: tests run in GB locale versions",
+    "noxvfb: tests that should not run in xvfb sessions"
+]
+testpaths = [
+    "tests"
+]
+addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json --reruns 3 --reruns-delay 2 -m 'not unstable and not noxvfb and headed' --html=artifacts/report_headed.html"
+
+[tool.ruff]
+target-version = "py310"

collect_executables.py

@@ -83,7 +83,7 @@ def get_gd_platform():
         gd_platform = get_gd_platform()
         ext = "zip" if "win" in gd_platform else "tar.gz"
         print(
-            f"https://github.com/mozilla/geckodriver/releases/download/v0.35.0/geckodriver-v0.35.0-{gd_platform}.{ext}"
+            f"https://github.com/mozilla/geckodriver/releases/download/v0.36.0/geckodriver-v0.36.0-{gd_platform}.{ext}"
         )
         exit()