WIP: modify tc files, add python script to run tests independent of os

Pipfile

@@ -25,6 +25,7 @@ google-auth = "2.32.0"
 psutil = "<6.1"
 pytest-json-report = "==1.5.0"
 beautifulsoup4 = "4.12.3"
+mozinstall = "*"
 
 [dev-packages]
 werkzeug = "==3.0.3"

l10n_CM/run_l10n.py

@@ -13,7 +13,16 @@
 valid_flags = {"--run-headless", "-n", "--reruns", "--fx-executable", "--ci"}
 flag_with_parameter = {"-n", "--reruns"}
 valid_region = {"US", "CA", "DE", "FR"}
-valid_sites = {"demo", "amazon", "walmart", "mediamarkt", "lowes", "etsy", "calvinklein", "bestbuy"}
+valid_sites = {
+    "demo",
+    "amazon",
+    "walmart",
+    "mediamarkt",
+    "lowes",
+    "etsy",
+    "calvinklein",
+    "bestbuy",
+}
 live_sites = []
 
 LOCALHOST = "127.0.0.1"

run_tests.py

@@ -0,0 +1,85 @@
+import argparse
+import os
+import platform
+import sys
+from pathlib import Path
+from subprocess import check_output
+
+import mozinstall
+import pytest
+import requests
+
+LINUX_FX_EXEC = "./firefox/firefox"
+WIN_FX_EXEC = "C:\\Program Files\\Custom Firefox\\firefox.exe"
+MAC_FX_EXEC = ""
+
+
+def get_fx_exec():
+    fx_exec = LINUX_FX_EXEC
+    if platform.system().lower().startswith("win"):
+        fx_exec = WIN_FX_EXEC
+    elif platform.system().lower().startswith("darwin"):
+        fx_exec = MAC_FX_EXEC
+    return fx_exec
+
+
+def install():
+    command = ["pipenv", "run", "python", "collect_executables.py"]
+    if not platform.system().lower().startswith("win"):
+        command = ["./collect_executables.sh"]
+    target_filename = "setup.exe"
+    if platform.system().lower().startswith("darwin"):
+        target_filename = "Firefox.dmg"
+    elif platform.system().lower().startswith("linux"):
+        target_filename = "firefox.tar.xz"
+    url = check_output(command).decode().strip()
+    if "collect_executables.py" in command:
+        resp = requests.get(url)
+        resp.raise_for_status()
+        with open(target_filename, "wb") as fh:
+            fh.write(resp.content)
+        installdir = Path(get_fx_exec()).parent
+        if not installdir.is_dir():
+            os.makedirs(installdir, exist_ok=True)
+        mozinstall.install(target_filename, installdir)
+
+
+def run_suite(parsed_args):
+    """Convert script / argparse args (parsed_args) to pytest_args"""
+    pytest_args = parsed_args.added_args or []
+    if parsed_args.install:
+        install()
+    if parsed_args.ci and "--fx-executable" not in pytest_args:
+        pytest_args.extend(["--fx-executable", get_fx_exec()])
+    if parsed_args.pyproject:
+        os.replace(parsed_args.pyproject, "pyproject.toml")
+    if parsed_args.subset:
+        tests = open("selected_tests").read()
+    else:
+        tests = "tests"
+    workers = None
+    if not parsed_args.headed:
+        workers = "auto"
+        pytest_args.append("--run-headless")
+    if parsed_args.workers:
+        workers = pytest_args.workers
+    if workers:
+        pytest_args.extend(["-n", workers])
+    pytest_args.extend(tests.split())
+    print("pytest " + " ".join(pytest_args))
+    return pytest.main(pytest_args)
+
+
+parser = argparse.ArgumentParser(
+    prog="Run STARfox tests", description="Run the STARfox suites"
+)
+
+parser.add_argument("added_args", nargs="*")
+parser.add_argument("--pyproject", default=None)
+parser.add_argument("-w", "--workers", default=None)
+parser.add_argument("-s", "--subset", action="store_true")
+parser.add_argument("--install", action="store_true")
+parser.add_argument("--ci", action="store_true")
+parser.add_argument("--headed", action="store_true")
+
+sys.exit(run_suite(parser.parse_args()))

taskcluster/config.yml

@@ -31,3 +31,8 @@ workers:
       implementation: generic-worker
       os: linux
       worker-type: 't-linux-2204-wayland'
+    b-win2022:
+      provisioner: '{trust-domain}-1'
+      implementation: generic-worker
+      os: windows
+      worker-type: 'b-win2022'

taskcluster/kinds/run-smoke-tests/kind.yml

@@ -3,9 +3,7 @@ transforms:
   - taskgraph.transforms.notify:transforms
 
 task-defaults:
-  label: "Smoke Tests"
   description: "Runs Smoke Tests and Notifies Slack"
-  worker-type: t-linux-wayland
   worker:
     taskcluster-proxy: true
     max-run-time: 1800
@@ -18,9 +16,71 @@ task-defaults:
   scopes:
     - queue:route:notify.slack-channel.C07AHPJ525V # notify mobile-alerts-sandbox on failure
     - notify:slack-channel:C07AHPJ525V
+  notify:
+    recipients:
+      - type: slack-channel
+        channel-id: C07AHPJ525V
+        status-type: on-defined
+    content:
+      slack:
+        blocks: [
+          {
+            "type": "header",
+            "text": {
+              "type": "plain_text",
+              "text": "firefox-desktop :firefox: ${task.metadata.name} Linux\n "
+            }
+          },
+          {
+            "type": "divider"
+          },
+          {
+            "type": "section",
+            "text": {
+              "type": "mrkdwn",
+              "text": "*Task*: <https://firefox-ci-tc.services.mozilla.com/tasks/${status.taskId}|Taskcluster>"
+            }
+          },
+          {
+            "type": "section",
+            "text": {
+              "type": "mrkdwn",
+              "text": "*Owner*: ${task.metadata.owner}"
+            }
+          },
+          {
+            "type": "section",
+            "text": {
+              "type": "mrkdwn",
+              "text": "*Commit*: <${task.metadata.source}>"
+            }
+          },
+          {
+            "type": "section",
+            "text": {
+              "type": "mrkdwn",
+              "text": "*Test Summary*: <https://firefoxci.taskcluster-artifacts.net/${status.taskId}/0/public/results/report.html?sort=result> :debug:"
+            }
+          },
+          {
+            "type": "divider"
+          },
+          {
+            "type": "context",
+            "elements": [
+              {
+                "type": "mrkdwn",
+                "text": ":testops-notify: created by Desktop QA Test Engineering"
+              }
+            ]
+          }
+        ]
+        text: "{task[name]} with id $taskId has finished!"
 
 tasks:
   linux:
+    label: "Smoke Tests Linux"
+    worker-type: t-linux-wayland
     run:
       using: run-task
       cwd: "{checkout}"
@@ -29,75 +89,33 @@ tasks:
         mkdir -p artifacts;
         pip3 install 'pipenv==2023.11.15';
         pip3 install 'ruff>=0.4.8,<0.5';
-        mv ./ci_pyproject.toml ./pyproject.toml;
         pipenv install;
         ./collect_executables.sh;
         ./firefox/firefox --version;
         . ./keyring-unlock.sh
         pipenv run python3 choose_ci_set.py
-        pipenv run pytest --fx-executable ./firefox/firefox -n 4 $(cat selected_tests)
+        pipenv run python3 run_tests.py -s --ci --pyproject ci_pyproject.toml
         export FAILURE=${?}
-        mv ./ci_pyproject_headed.toml ./pyproject.toml;
         pipenv run python3 choose_ci_set.py
-        pipenv run pytest --fx-executable ./firefox/firefox $(cat selected_tests)
+        pipenv run python3 run_tests.py -s --ci --headed --pyproject ci_pyproject_headed.toml
         exit $((${?} | ${FAILURE}))
-    notify:
-      recipients:
-        - type: slack-channel
-          channel-id: C07AHPJ525V
-          status-type: on-defined
-      content:
-        slack:
-          blocks: [
-            {
-              "type": "header",
-              "text": {
-                "type": "plain_text",
-                "text": "firefox-desktop :firefox: ${task.metadata.name} Linux\n "
-              }
-            },
-            {
-              "type": "divider"
-            },
-            {
-              "type": "section",
-              "text": {
-                "type": "mrkdwn",
-                "text": "*Task*: <https://firefox-ci-tc.services.mozilla.com/tasks/${status.taskId}|Taskcluster>"
-              }
-            },
-            {
-              "type": "section",
-              "text": {
-                "type": "mrkdwn",
-                "text": "*Owner*: ${task.metadata.owner}"
-              }
-            },
-            {
-              "type": "section",
-              "text": {
-                "type": "mrkdwn",
-                "text": "*Commit*: <${task.metadata.source}>"
-              }
-            },
-            {
-              "type": "section",
-              "text": {
-                "type": "mrkdwn",
-                "text": "*Test Summary*: <https://firefoxci.taskcluster-artifacts.net/${status.taskId}/0/public/results/report.html?sort=result> :debug:"
-              }
-            },
-            {
-              "type": "divider"
-            },
-            {
-              "type": "context",
-              "elements": [
-                {
-                  "type": "mrkdwn",
-                  "text": ":testops-notify: created by Desktop QA Test Engineering"
-                }
-              ]
-            }
-          ]
-          text: "{task[name]} with id $taskId has finished!"
+
+  windows:
+    label: "Smoke Tests Windows"
+    worker-type: b-win2022
+    run:
+      using: run-task
+      cwd: "{checkout}"
+      command: |-
+        mkdir -p artifacts;
+        python --version
+        python -m pip install pipenv==2023.11.15
+        cd build\src
+        dir .
+        set PATH=%PATH%;C:\mozilla-build\python3\scripts
+        pipenv install
+        pipenv run python choose_ci_set.py
+        pipenv run pip list
+        pipenv run python -c "import requests"
+        pipenv run python run_tests.py -s --ci --install --pyproject ci_pyproject.toml
+        pipenv run python run_tests.py -s --ci --headed --pyproject ci_pyproject_headed.toml