Skip to content

Commit 17606f8

Browse files
ben-c-at-mozben-c-at-moz
authored
run headless and headed tests (#157)
* run headless and headed tests * pynput in mac and win * debug workflow * more reruns for headed * fix win path for artifacts * mark unstable test; change mark name * typo * force hide the menu * fix win artifacts * fix win artifacts * fix win artifacts * fix powershell command
1 parent 5d60cdc commit 17606f8

File tree

11 files changed

+107
-61
lines changed

11 files changed

+107
-61
lines changed

‎.github/workflows/smoke.yml

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -26,15 +26,27 @@ jobs:
2626
mv ./ci_pyproject.toml ./pyproject.toml;
2727
pipenv install;
2828
- name: Run Setup.exe
29+
id: setup
2930
run: |
3031
.\setup.exe /silent;
3132
Start-Sleep -Seconds 10;
3233
- name: Run Smoke Tests in Win
34+
if: steps.setup.conclusion == 'success'
3335
run: |
3436
pipenv run pytest -n 4 .
3537
$env:TEST_EXIT_CODE = $LASTEXITCODE
3638
mv artifacts artifacts-win || true
3739
exit $env:TEST_EXIT_CODE
40+
- name: Run Smoke Tests in Win (Headed)
41+
if: steps.setup.conclusion == 'success'
42+
run: |
43+
rm ./pyproject.toml;
44+
mv ./ci_pyproject_headed.toml ./pyproject.toml;
45+
pipenv run pytest -n 4 .
46+
$env:TEST_EXIT_CODE = $LASTEXITCODE
47+
rm artifacts/assets -r -Force
48+
mv artifacts/* artifacts-win
49+
exit $env:TEST_EXIT_CODE
3850
- name: Upload artifacts
3951
if: always()
4052
uses: actions/upload-artifact@v4
@@ -65,6 +77,12 @@ jobs:
6577
pipenv run pytest --fx-executable $(cat ./fx_location) -s -n 4 . || TEST_EXIT_CODE=$?
6678
mv artifacts artifacts-mac || true
6779
exit $TEST_EXIT_CODE
80+
- name: Run Smoke Tests in MacOS (Headed)
81+
run: |
82+
mv ./ci_pyproject_headed.toml ./pyproject.toml;
83+
pipenv run pytest --fx-executable $(cat ./fx_location) -s -n 4 . || TEST_EXIT_CODE=$?
84+
mv -r artifacts/* artifacts-mac/ || true
85+
exit $TEST_EXIT_CODE
6886
- name: Upload artifacts
6987
if: always()
7088
uses: actions/upload-artifact@v4

‎ci_pyproject.toml

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@ log_cli = true
44
log_cli_level = "warn"
55
markers = [
66
"audio: test is reliant on audio",
7-
"pynput: test uses pynput package",
7+
"headed: test must run in headed mode (e.g. uses pynput)",
88
"incident: incident smoke tests",
99
"unstable: temporary mark for unstable tests",
1010
"slow: test is clocked at more than 30s on modern machines",
@@ -16,7 +16,7 @@ markers = [
1616
testpaths = [
1717
"tests"
1818
]
19-
addopts = "-vs --ci --reruns 2 --reruns-delay 1 -m 'not incident and not unstable' --html=artifacts/report.html"
19+
addopts = "-vs --ci --run-headless --reruns 2 --reruns-delay 1 -m 'not incident and not unstable and not headed' --html=artifacts/report.html"
2020

2121
[tool.ruff]
2222
target-version = "py310"

‎ci_pyproject_headed.toml

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,22 @@
1+
[tool.pytest.ini_options]
2+
generate_report_on_test = true
3+
log_cli = true
4+
log_cli_level = "warn"
5+
markers = [
6+
"audio: test is reliant on audio",
7+
"headed: test must run in headed mode (e.g. uses pynput)",
8+
"incident: incident smoke tests",
9+
"unstable: temporary mark for unstable tests",
10+
"slow: test is clocked at more than 30s on modern machines",
11+
"ci: basic tests to run in ci",
12+
"locale_de: tests run in DE locale versions",
13+
"locale_fr: tests run in FR locale versions",
14+
"locale_gb: tests run in GB locale versions"
15+
]
16+
testpaths = [
17+
"tests"
18+
]
19+
addopts = "-vs --ci --reruns 4 --reruns-delay 1 -m 'not incident and not unstable and headed' --html=artifacts/report_headed.html"
20+
21+
[tool.ruff]
22+
target-version = "py310"

‎conftest.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -250,7 +250,7 @@ def driver(
250250
"""
251251
try:
252252
options = Options()
253-
if opt_headless or opt_ci:
253+
if opt_headless:
254254
options.add_argument("--headless")
255255
options.binary_location = fx_executable
256256
for opt, value in set_prefs:

‎dev_pyproject.toml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,7 +5,7 @@ log_cli = true
55
log_cli_level = "info"
66
markers = [
77
"audio: test is reliant on audio",
8-
"pynput: test uses pynput package",
8+
"headed: test must run in headed mode (e.g. uses pynput)",
99
"incident: incident smoke tests",
1010
"unstable: temporary mark for unstable tests",
1111
"slow: test is clocked at more than 30s on modern machines",

‎pyproject.toml

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,11 +1,10 @@
11
[tool.pytest.ini_options]
2-
generate_report_on_test = false
32
addopts = "-vs --html=report.html"
43
log_cli = true
54
log_cli_level = "info"
65
markers = [
76
"audio: test is reliant on audio",
8-
"pynput: test uses pynput package",
7+
"headed: test must run in headed mode (e.g. uses pynput)",
98
"incident: incident smoke tests",
109
"unstable: temporary mark for unstable tests",
1110
"slow: test is clocked at more than 30s on modern machines",

‎run_commit_tests.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,7 @@
2424
"pytest",
2525
"--run-headless",
2626
"-m",
27-
"not unstable",
27+
"not unstable and not headed",
2828
"-n",
2929
"4",
3030
*tests,

‎taskcluster/kinds/run-smoke-tests/kind.yml

Lines changed: 53 additions & 51 deletions
Original file line numberDiff line numberDiff line change
@@ -31,6 +31,8 @@ tasks:
3131
./collect_executables.sh;
3232
mv ./ci_pyproject.toml ./pyproject.toml;
3333
pipenv install;
34+
pipenv run pytest --fx-executable $(cat ./fx_location) -n 4 .;
35+
mv ./ci_pyproject_headed.toml ./pyproject.toml;
3436
pipenv run pytest --fx-executable $(cat ./fx_location) -n 4 .
3537
notify:
3638
recipients:
@@ -40,55 +42,55 @@ tasks:
4042
content:
4143
slack:
4244
blocks: [
43-
{
44-
"type": "header",
45-
"text": {
46-
"type": "plain_text",
47-
"text": "firefox-desktop :firefox: ${task.metadata.name}\n "
48-
}
49-
},
50-
{
51-
"type": "divider"
52-
},
53-
{
54-
"type": "section",
55-
"text": {
56-
"type": "mrkdwn",
57-
"text": "*Task*: <https://firefox-ci-tc.services.mozilla.com/tasks/${status.taskId}|Taskcluster>"
58-
}
59-
},
60-
{
61-
"type": "section",
62-
"text": {
63-
"type": "mrkdwn",
64-
"text": "*Owner*: ${task.metadata.owner}"
65-
}
66-
},
67-
{
68-
"type": "section",
69-
"text": {
70-
"type": "mrkdwn",
71-
"text": "*Commit*: <${task.metadata.source}>"
72-
}
73-
},
74-
{
75-
"type": "section",
76-
"text": {
77-
"type": "mrkdwn",
78-
"text": "*Test Summary*: <https://firefoxci.taskcluster-artifacts.net/${status.taskId}/0/public/results/report.html?sort=result> :debug:"
79-
}
80-
},
81-
{
82-
"type": "divider"
83-
},
84-
{
85-
"type": "context",
86-
"elements": [
87-
{
88-
"type": "mrkdwn",
89-
"text": ":testops-notify: created by Desktop QA Test Engineering"
90-
}
91-
]
92-
}
93-
]
45+
{
46+
"type": "header",
47+
"text": {
48+
"type": "plain_text",
49+
"text": "firefox-desktop :firefox: ${task.metadata.name}\n "
50+
}
51+
},
52+
{
53+
"type": "divider"
54+
},
55+
{
56+
"type": "section",
57+
"text": {
58+
"type": "mrkdwn",
59+
"text": "*Task*: <https://firefox-ci-tc.services.mozilla.com/tasks/${status.taskId}|Taskcluster>"
60+
}
61+
},
62+
{
63+
"type": "section",
64+
"text": {
65+
"type": "mrkdwn",
66+
"text": "*Owner*: ${task.metadata.owner}"
67+
}
68+
},
69+
{
70+
"type": "section",
71+
"text": {
72+
"type": "mrkdwn",
73+
"text": "*Commit*: <${task.metadata.source}>"
74+
}
75+
},
76+
{
77+
"type": "section",
78+
"text": {
79+
"type": "mrkdwn",
80+
"text": "*Test Summary*: <https://firefoxci.taskcluster-artifacts.net/${status.taskId}/0/public/results/report.html?sort=result> :debug:"
81+
}
82+
},
83+
{
84+
"type": "divider"
85+
},
86+
{
87+
"type": "context",
88+
"elements": [
89+
{
90+
"type": "mrkdwn",
91+
"text": ":testops-notify: created by Desktop QA Test Engineering"
92+
}
93+
]
94+
}
95+
]
9496
text: "{task[name]} with id $taskId has finished!"

‎tests/incident/test_pdf_form_input.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,7 +14,7 @@ def test_url():
1414

1515

1616
@pytest.mark.incident
17-
@pytest.mark.pynput
17+
@pytest.mark.headed
1818
def test_pdf_form_fill(driver, test_url):
1919
try:
2020
from pynput.keyboard import Controller, Key

‎tests/menus/test_frequently_used_context_menu.py

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,6 @@
11
import platform
2+
import sys
3+
from os import environ
24
from time import sleep
35

46
import pytest
@@ -8,8 +10,11 @@
810
from modules.page_object import ExamplePage
911
from modules.util import BrowserActions, Utilities
1012

13+
WIN_GHA = environ.get("GITHUB_ACTIONS") == "true" and sys.platform.startswith("win")
1114

12-
@pytest.mark.unstable
15+
16+
@pytest.mark.skipif(WIN_GHA, reason="Test unstable in Windows Github Actions")
17+
@pytest.mark.headed
1318
def test_save_page_as(driver: Firefox):
1419
"""
1520
C2637623.1: save page as

0 commit comments

Comments
 (0)