mark tests that fail in xvfb sessions

Author: ben-c-at-moz

.github/workflows/smoke.yml

@@ -255,7 +255,7 @@ jobs:
           pip3 install 'pipenv==2023.11.15';
           pip3 install 'ruff>=0.4.8,<0.5';
           rm ./pyproject.toml;
-          mv ./ci_pyproject.toml ./pyproject.toml;
+          mv ./ci_xvfb_pyproject.toml ./pyproject.toml;
           pipenv install;
           ./collect_executables.sh
           ./firefox/firefox --version;
@@ -275,7 +275,7 @@ jobs:
         env:
           FX_EXECUTABLE: ./firefox/firefox
         run: |
-          mv ./ci_pyproject_headed.toml ./pyproject.toml;
+          mv ./ci_xvfb_pyproject_headed.toml ./pyproject.toml;
           pipenv run python choose_ci_set.py
           DISPLAY=:99 pipenv run pytest --fx-executable="$FX_EXECUTABLE" $(cat selected_tests) || TEST_EXIT_CODE=$?
           exit $TEST_EXIT_CODE

ci_xvfb_pyproject.toml

@@ -0,0 +1,23 @@
+[tool.pytest.ini_options]
+generate_report_on_test = true
+log_cli = true
+log_cli_level = "warn"
+markers = [
+    "audio: test is reliant on audio",
+    "headed: test must run in headed mode (e.g. uses pynput)",
+    "incident: incident smoke tests",
+    "unstable: temporary mark for unstable tests",
+    "slow: test is clocked at more than 30s on modern machines",
+    "ci: basic tests to run in ci",
+    "locale_de: tests run in DE locale versions",
+    "locale_fr: tests run in FR locale versions",
+    "locale_gb: tests run in GB locale versions",
+    "noxvfb: tests that should not run in xvfb sessions"
+]
+testpaths = [
+    "tests"
+]
+addopts = "-vs --ci --run-headless --json-report --json-report-file artifacts/report.json --reruns 3 --reruns-delay 3 -m 'not unstable and not headed and not noxvfb' --html=artifacts/report.html"
+
+[tool.ruff]
+target-version = "py310"

ci_xvfb_pyproject_headed.toml

@@ -0,0 +1,22 @@
+[tool.pytest.ini_options]
+generate_report_on_test = true
+log_cli = true
+log_cli_level = "warn"
+markers = [
+    "audio: test is reliant on audio",
+    "headed: test must run in headed mode (e.g. uses pynput)",
+    "incident: incident smoke tests",
+    "unstable: temporary mark for unstable tests",
+    "slow: test is clocked at more than 30s on modern machines",
+    "ci: basic tests to run in ci",
+    "locale_de: tests run in DE locale versions",
+    "locale_fr: tests run in FR locale versions",
+    "locale_gb: tests run in GB locale versions"
+]
+testpaths = [
+    "tests"
+]
+addopts = "-vs --ci --json-report --json-report-file artifacts/report_headed.json --reruns 3 --reruns-delay 2 -m 'not unstable and not noxvfb and headed' --html=artifacts/report_headed.html"
+
+[tool.ruff]
+target-version = "py310"

tests/address_bar_and_search/test_addon_suggestion.py

@@ -31,6 +31,7 @@ def add_to_prefs_list():
     ]
 
 
+@pytest.mark.noxvfb
 def test_addon_suggestion_based_on_search_input(driver: Firefox):
     """
     C2234714 - Verify that the address bar suggests relevant add-ons based on search input.

tests/audio_video/test_allow_audio_video_functionality.py

@@ -23,6 +23,7 @@ def test_case():
 
 @pytest.mark.skipif(WIN_GHA, reason="Test unstable in Windows Github Actions")
 @pytest.mark.audio
+@pytest.mark.noxvfb
 def test_allow_audio_video_functionality(driver: Firefox):
     """
     C330155 : 'Allow Audio and Video' functionality

tests/downloads/test_add_mime_type_doc.py

@@ -25,6 +25,7 @@ def delete_files_regex_string():
 
 
 @pytest.mark.skipif(WIN_GHA, reason="Test unstable in Windows Github Actions")
+@pytest.mark.noxvfb
 def test_mime_type_doc(driver: Firefox, sys_platform: str, opt_ci: bool, delete_files):
     """
     C1756748: Verify the user can add the .doc type

tests/password_manager/test_password_csv_correctness.py

@@ -16,6 +16,7 @@ def test_case():
 
 
 @pytest.mark.headed
+@pytest.mark.noxvfb
 def test_password_csv_correctness(
     driver_and_saved_logins, downloads_folder, sys_platform
 ):

tests/password_manager/test_password_csv_export.py

@@ -15,6 +15,7 @@ def test_case():
 
 
 @pytest.mark.headed
+@pytest.mark.noxvfb
 def test_password_csv_export(
     driver_and_saved_logins, downloads_folder, sys_platform, opt_ci
 ):

tests/pdf_viewer/test_add_image_pdf.py

@@ -33,6 +33,7 @@ def add_to_prefs_list():
 
 
 @pytest.mark.headed
+@pytest.mark.noxvfb
 def test_add_image_pdf(driver: Firefox, sys_platform, pdf_viewer: GenericPdf):
     """
     C2228202: Verify that the user is able to add an image to a PDF file.

tests/pdf_viewer/test_pdf_navigation.py

@@ -144,6 +144,7 @@ def test_toolbar_options_cursor(driver: Firefox, pdf_viewer: GenericPdf):
     assert cursor_style == "auto"
 
 
+@pytest.mark.noxvfb
 def test_toolbar_options_rotate_cw(driver: Firefox, pdf_viewer: GenericPdf):
     """
     C3927.5: Ensure the correct rotation is shown
@@ -162,6 +163,7 @@ def test_toolbar_options_rotate_cw(driver: Firefox, pdf_viewer: GenericPdf):
         )
 
 
+@pytest.mark.noxvfb
 def test_toolbar_options_rotate_ccw(driver: Firefox, pdf_viewer: GenericPdf):
     """
     C3927.5: Ensure the correct rotation is shown