Attempt to fix the "ghost runs" issue in scheduled beta (#473)

ben-c-at-moz · web-flow · commit 239b0dee5191 · 2025-02-18T13:42:25.000-08:00
* bomb out if there is no config; don't count empty runs for reportable

* update ruff version
diff --git a/Pipfile b/Pipfile
@@ -17,7 +17,7 @@ taskcluster-taskgraph = "==9.0.0"
 jsonpath-ng = "1.6.1"
 pillow = "<10.5"
 pyfxa = "0.7.0"
-ruff = "0.5.2"
+ruff = "0.9.6"
 pytest-rerunfailures = "14.0"
 slack-sdk = "3.31.0"
 google-cloud-storage = "2.18.0"
diff --git a/collect_executables.py b/collect_executables.py
@@ -111,23 +111,23 @@ def get_gd_platform():
         minor = int(minor)
         beta = int(beta)
 
-        next_major = f"{major+1}.0b1"
+        next_major = f"{major + 1}.0b1"
         fx_download_dir_url = f"https://archive.mozilla.org/pub/firefox/candidates/{next_major}-candidates/build1/"
         rs = requests.get(fx_download_dir_url)
         if rs.status_code < 300:
             latest_beta_ver = next_major
             this_beta = next_major
             continue
 
-        next_minor = f"{major}.{minor+1}b1"
+        next_minor = f"{major}.{minor + 1}b1"
         fx_download_dir_url = f"https://archive.mozilla.org/pub/firefox/candidates/{next_minor}-candidates/build1/"
         rs = requests.get(fx_download_dir_url)
         if rs.status_code < 300:
             latest_beta_ver = next_minor
             this_beta = next_minor
             continue
 
-        next_beta = f"{major}.{minor}b{beta+1}"
+        next_beta = f"{major}.{minor}b{beta + 1}"
         fx_download_dir_url = f"https://archive.mozilla.org/pub/firefox/candidates/{next_beta}-candidates/build1/"
         rs = requests.get(fx_download_dir_url)
         if rs.status_code < 300:
diff --git a/l10n_CM/Unified/test_demo_ad_address_data_captured_in_doorhanger_and_stored.py b/l10n_CM/Unified/test_demo_ad_address_data_captured_in_doorhanger_and_stored.py
@@ -89,4 +89,6 @@ def test_demo_ad_address_data_captured_in_doorhanger_and_stored(
     found_address_data = any(
         all(value in element.text for value in expected_values) for element in elements
     )
-    assert found_address_data, "Street, city, state (if applicable), zip, or country were not found in any of the address entries!"
+    assert found_address_data, (
+        "Street, city, state (if applicable), zip, or country were not found in any of the address entries!"
+    )
diff --git a/l10n_CM/Unified/test_demo_ad_email_phone_captured_in_doorhanger_and_stored.py b/l10n_CM/Unified/test_demo_ad_email_phone_captured_in_doorhanger_and_stored.py
@@ -4,16 +4,17 @@
 from modules.browser_object_autofill_popup import AutofillPopup
 from modules.page_object_autofill import AddressFill
 from modules.page_object_prefs import AboutPrefs
-from modules.util import Utilities, BrowserActions
+from modules.util import BrowserActions, Utilities
 
 
 @pytest.fixture()
 def test_case():
     return "2886581"
 
 
-def test_demo_ad_email_phone_captured_in_doorhanger_and_stored(driver: Firefox, region: str
-                                                               ):
+def test_demo_ad_email_phone_captured_in_doorhanger_and_stored(
+    driver: Firefox, region: str
+):
     """
     C2888704 - Verify tele/email data are captured in the Capture Doorhanger and stored in about:preferences
     """
@@ -38,7 +39,9 @@ def test_demo_ad_email_phone_captured_in_doorhanger_and_stored(driver: Firefox,
     # containing phone field
     expected_phone = address_autofill_data.telephone
     with driver.context(driver.CONTEXT_CHROME):
-        actual_phone = address_autofill_popup.get_element("address-doorhanger-phone").text
+        actual_phone = address_autofill_popup.get_element(
+            "address-doorhanger-phone"
+        ).text
     normalize_expected = util.normalize_phone_number(expected_phone)
     normalized_actual = util.normalize_phone_number(actual_phone)
     assert normalized_actual == normalize_expected
@@ -55,9 +58,8 @@ def test_demo_ad_email_phone_captured_in_doorhanger_and_stored(driver: Firefox,
     elements = about_prefs.get_elements("saved-addresses-values")
     expected_values = [expected_phone, expected_email]
     found_email_phone = any(
-        all(value in element.text for value in expected_values)
-        for element in elements
+        all(value in element.text for value in expected_values) for element in elements
+    )
+    assert found_email_phone, (
+        "Email or phone were not found in any of the address entries!"
     )
-    assert (
-        found_email_phone
-    ), "Email or phone were not found in any of the address entries!"
diff --git a/l10n_CM/Unified/test_demo_ad_name_org_captured_in_doorhanger_and_stored.py b/l10n_CM/Unified/test_demo_ad_name_org_captured_in_doorhanger_and_stored.py
@@ -59,6 +59,6 @@ def test_demo_ad_name_org_captured_in_doorhanger_and_stored(
     found_name_org = any(
         all(value in element.text for value in expected_values) for element in elements
     )
-    assert (
-        found_name_org
-    ), "Name or organization were not found in any of the address entries!"
+    assert found_name_org, (
+        "Name or organization were not found in any of the address entries!"
+    )
diff --git a/modules/browser_object_tracker_panel.py b/modules/browser_object_tracker_panel.py
@@ -145,7 +145,9 @@ def verify_tracker_shield_indicator(self, nav: Navigation) -> BasePage:
             assert (
                 shield_icon.get_attribute("data-l10n-id")
                 == "tracking-protection-icon-active-container"
-            ), "The label detected did not correspond to the expected one: tracking-protection-icon-active-container"
+            ), (
+                "The label detected did not correspond to the expected one: tracking-protection-icon-active-container"
+            )
         return self
 
     def open_and_return_cross_site_trackers(self) -> List[WebElement]:
diff --git a/modules/page_base.py b/modules/page_base.py
@@ -437,9 +437,9 @@ def verify_opened_image_url(self, url_substr: str, pattern: str) -> Page:
         self.url_contains(url_substr)
         current_url = self.driver.current_url
 
-        assert re.match(
-            pattern, current_url
-        ), f"URL does not match the expected pattern: {current_url}"
+        assert re.match(pattern, current_url), (
+            f"URL does not match the expected pattern: {current_url}"
+        )
         return self
 
     def fill(
@@ -490,9 +490,9 @@ def fetch(self, reference: Union[str, tuple, WebElement], labels=[]) -> WebEleme
             return self.find_element(*reference)
         elif isinstance(reference, WebElement):
             return reference
-        assert (
-            False
-        ), "Bad fetch: only selectors, selector names, or WebElements allowed."
+        assert False, (
+            "Bad fetch: only selectors, selector names, or WebElements allowed."
+        )
 
     def click_on(self, reference: Union[str, tuple, WebElement], labels=[]) -> Page:
         """Click on an element, no matter the context, return the page"""
diff --git a/modules/testrail.py b/modules/testrail.py
@@ -351,7 +351,9 @@ def matching_configs(self, testrail_project_id, config_group_id, config_name):
         matching_group = next(c for c in configs if c.get("id") == config_group_id)
         logging.info(f"matching group|| {matching_group}")
         cfgs = [
-            c for c in matching_group.get("configs") if config_name in c.get("name")
+            c
+            for c in matching_group.get("configs")
+            if config_name == c.get("name").strip()
         ]
         return cfgs
 
diff --git a/modules/testrail_integration.py b/modules/testrail_integration.py
@@ -124,7 +124,7 @@ def reportable():
     covered_suites = 0
     for entry in plan_entries:
         for run_ in entry.get("runs"):
-            if platform in run_.get("config"):
+            if run_.get("config") and platform in run_.get("config"):
                 covered_suites += 1
 
     num_suites = 0
@@ -378,6 +378,10 @@ def collect_changes(testrail_session: TestRail, report):
         release = ".".join(release.split(".")[:-1])
         config = f"{os_name} {release} {arch}"
 
+    logging.warning(f"Reporting for config: {config}")
+    if not config.strip():
+        raise ValueError("Config cannot be blank.")
+
     with open(".tmp_testrail_info", "w") as fh:
         fh.write(f"{plan_title}|{config}")
 
@@ -421,8 +425,7 @@ def collect_changes(testrail_session: TestRail, report):
             logging.info("Creating config...")
             testrail_session.add_config(CONFIG_GROUP_ID, config)
         tried = True
-    if len(config_matches) >= 1:
-        # TODO: change above to == 1
+    if len(config_matches) == 1:
         config_id = config_matches[0].get("id")
         logging.info(f"config id: {config_id}")
     else:
diff --git a/tests/address_bar_and_search/test_adaptive_history_autofill.py b/tests/address_bar_and_search/test_adaptive_history_autofill.py
@@ -61,11 +61,11 @@ def test_add_adaptive_history_autofill(driver: Firefox):
     )
 
     # Assertion to verify that the 'autofill_adaptive' type is found
-    assert (
-        autofill_adaptive_element.get_attribute("type") == "autofill_adaptive"
-    ), f"Expected element type to be 'autofill_adaptive' but found '{autofill_adaptive_element.get_attribute('type')}'"
+    assert autofill_adaptive_element.get_attribute("type") == "autofill_adaptive", (
+        f"Expected element type to be 'autofill_adaptive' but found '{autofill_adaptive_element.get_attribute('type')}'"
+    )
 
     # Assertion to check the autofilled URL is the expected one
-    assert (
-        "nationalgeographic.com/science" in autofill_adaptive_element.text
-    ), "URL 'https://www.nationalgeographic.com/science' not found in autofill suggestions."
+    assert "nationalgeographic.com/science" in autofill_adaptive_element.text, (
+        "URL 'https://www.nationalgeographic.com/science' not found in autofill suggestions."
+    )
diff --git a/tests/address_bar_and_search/test_preferences_all_toggles_enabled.py b/tests/address_bar_and_search/test_preferences_all_toggles_enabled.py
@@ -29,13 +29,13 @@ def test_preferences_all_toggles_enabled(driver: Firefox):
     if not show_suggestions_checkbox.is_selected():
         show_suggestions_checkbox.click()
     nonsponsored_checkbox = about_prefs.get_element("firefox-suggest-nonsponsored")
-    assert (
-        nonsponsored_checkbox.is_selected()
-    ), f"Checkbox with selector '{nonsponsored_checkbox}' is not checked"
+    assert nonsponsored_checkbox.is_selected(), (
+        f"Checkbox with selector '{nonsponsored_checkbox}' is not checked"
+    )
     sponsors_checkbox = about_prefs.get_element("firefox-suggest-sponsored")
-    assert (
-        sponsors_checkbox.is_selected()
-    ), f"Checkbox with selector '{sponsors_checkbox}' is not checked"
+    assert sponsors_checkbox.is_selected(), (
+        f"Checkbox with selector '{sponsors_checkbox}' is not checked"
+    )
 
     # Check if sponsored suggestion is displayed. Keep checking until the sponsered suggestions are displayed
     found = False
diff --git a/tests/address_bar_and_search/test_tile_menu_options.py b/tests/address_bar_and_search/test_tile_menu_options.py
@@ -138,6 +138,6 @@ def test_tile_context_menu_options(driver: Firefox, index: int, sponsored: bool)
             set_in_use.remove(match)
             logging.info(f"Detected the context item: {match}")
 
-    assert (
-        len(set_in_use) == 0
-    ), "Did not find all of the required context menu actions."
+    assert len(set_in_use) == 0, (
+        "Did not find all of the required context menu actions."
+    )
diff --git a/tests/bookmarks_and_history/test_opened_website_present_in_toolbar_history.py b/tests/bookmarks_and_history/test_opened_website_present_in_toolbar_history.py
@@ -32,6 +32,6 @@ def test_the_most_recent_website_is_present_in_history_menu(driver: Firefox):
     # Verify YouTube is present in the history menu and is on top of the list as the most recent website visited
     with driver.context(driver.CONTEXT_CHROME):
         recent_history_elements = panel_ui.get_elements("recent-history-content")
-        assert (
-            recent_history_elements[0].get_attribute("value") == "YouTube"
-        ), "YouTube is not the first item in the recent history."
+        assert recent_history_elements[0].get_attribute("value") == "YouTube", (
+            "YouTube is not the first item in the recent history."
+        )
diff --git a/tests/bookmarks_and_history/test_private_window_website_not_in_history.py b/tests/bookmarks_and_history/test_private_window_website_not_in_history.py
@@ -30,6 +30,6 @@ def test_opened_website_in_private_window_not_captured_in_history_list(driver: F
         empty_label = panel_ui.get_element("recent-history-content").get_attribute(
             "value"
         )
-        assert (
-            empty_label == "(Empty)"
-        ), f"Expected history to be empty, but found '{empty_label}'"
+        assert empty_label == "(Empty)", (
+            f"Expected history to be empty, but found '{empty_label}'"
+        )
diff --git a/tests/downloads/test_download_pdf.py b/tests/downloads/test_download_pdf.py
@@ -48,9 +48,9 @@ def test_download_pdf(
     saved_pdf_location = os.path.join(downloads_folder, file_name)
 
     # Verify if the file exists
-    assert os.path.exists(
-        saved_pdf_location
-    ), f"The file was not downloaded to {saved_pdf_location}."
+    assert os.path.exists(saved_pdf_location), (
+        f"The file was not downloaded to {saved_pdf_location}."
+    )
 
     print(
         f"Test passed: The file {file_name} has been downloaded and is present at {saved_pdf_location}."
diff --git a/tests/downloads/test_mixed_content_download_via_https.py b/tests/downloads/test_mixed_content_download_via_https.py
@@ -49,12 +49,12 @@ def test_mixed_content_download_via_https(driver: Firefox, delete_files):
 
         # Verify that the desired download target element is present directly, no extra steps needed.
         download_value = download_name.get_attribute("value")
-        assert re.match(
-            r"download(\(\d+\))?$", download_value
-        ), f"The download name is incorrect: {download_value}"
+        assert re.match(r"download(\(\d+\))?$", download_value), (
+            f"The download name is incorrect: {download_value}"
+        )
 
         # Verify that the download progress has reached 100%, which indicates that the download is complete.
         download_status_value = download_status.get_attribute("value")
-        assert (
-            download_status_value == "100"
-        ), f"The download status is not '100': {download_status_value}"
+        assert download_status_value == "100", (
+            f"The download status is not '100': {download_status_value}"
+        )
diff --git a/tests/form_autofill/test_address_autofill_attribute.py b/tests/form_autofill/test_address_autofill_attribute.py
@@ -39,6 +39,6 @@ def test_address_attribute_selection(driver: Firefox):
     # Get the primary value (street address) from the first item in the dropdown and assert that the actual value
     # matches the expected value
     expected_street_address = autofill_sample_data.street_address
-    assert (
-        expected_street_address == actual_value
-    ), f"Expected {expected_street_address}, but got {actual_value}"
+    assert expected_street_address == actual_value, (
+        f"Expected {expected_street_address}, but got {actual_value}"
+    )
diff --git a/tests/form_autofill/test_name_autofill_attribute.py b/tests/form_autofill/test_name_autofill_attribute.py
@@ -39,6 +39,6 @@ def test_name_attribute_selection(driver: Firefox):
     # Get the primary value (street address) from the first item in the dropdown and assert that the actual value
     # matches the expected value
     expected_name = autofill_sample_data.name
-    assert (
-        expected_name == actual_value
-    ), f"Expected {expected_name}, but got {actual_value}"
+    assert expected_name == actual_value, (
+        f"Expected {expected_name}, but got {actual_value}"
+    )
diff --git a/tests/form_autofill/test_updating_address.py b/tests/form_autofill/test_updating_address.py
@@ -48,17 +48,17 @@ def test_update_address(driver: Firefox):
 
     # Verify no dupe is saved
     element_list = about_prefs.get_elements("saved-addresses")
-    assert (
-        len(element_list) == 1
-    ), f"Expected 1 saved address, but found {len(element_list)}."
+    assert len(element_list) == 1, (
+        f"Expected 1 saved address, but found {len(element_list)}."
+    )
     print("Assertion passed: There is exactly 1 saved address.")
 
     # Assert that "Doe" is present in updated entry
     elements = about_prefs.get_elements("saved-addresses-values")
     found_updated_address = any("Doe" in element.text for element in elements)
-    assert (
-        found_updated_address
-    ), "The value 'Doe' was not found in any of the address entries."
+    assert found_updated_address, (
+        "The value 'Doe' was not found in any of the address entries."
+    )
 
     # Print the element that contains "Doe" for verification
     for element in elements:
diff --git a/tests/pdf_viewer/test_download_pdf_with_form_fields.py b/tests/pdf_viewer/test_download_pdf_with_form_fields.py
@@ -49,9 +49,9 @@ def test_download_pdf_with_form_fields(
     saved_pdf_location = os.path.join(downloads_folder, file_name)
 
     # Verify if the file exists
-    assert os.path.exists(
-        saved_pdf_location
-    ), f"The file was not downloaded to {saved_pdf_location}."
+    assert os.path.exists(saved_pdf_location), (
+        f"The file was not downloaded to {saved_pdf_location}."
+    )
 
     # Open the saved pdf and check if the edited field is displayed
     driver.get("file://" + os.path.realpath(saved_pdf_location))
diff --git a/tests/pdf_viewer/test_download_triggered_on_content_disposition_attachment.py b/tests/pdf_viewer/test_download_triggered_on_content_disposition_attachment.py
@@ -53,9 +53,9 @@ def test_download_panel_triggered_on_content_disposition_attachment(
         driver.find_element(By.ID, "unknownContentTypeWindow").send_keys(Keys.ENTER)
 
     tabs.wait_for_num_tabs(2)
-    assert (
-        len(driver.window_handles) == 2
-    ), f"Expected 2 tabs, but found {len(driver.window_handles)}"
+    assert len(driver.window_handles) == 2, (
+        f"Expected 2 tabs, but found {len(driver.window_handles)}"
+    )
 
     tabs.switch_to_new_tab()
     GenericPdf(driver, pdf_url="").url_contains("file:")
diff --git a/tests/pdf_viewer/test_pdf_data_can_be_cleared.py b/tests/pdf_viewer/test_pdf_data_can_be_cleared.py
@@ -71,7 +71,7 @@ def test_pdf_data_can_be_cleared(
     pdf.expect(lambda _: default_option.is_selected())
 
     # Assert that all fields are reset to their default state
-    assert (
-        pdf.get_element("first-name-field").get_attribute("value") == ""
-    ), "Text field did not reset."
+    assert pdf.get_element("first-name-field").get_attribute("value") == "", (
+        "Text field did not reset."
+    )
     assert not checkbox.is_selected(), "Checkbox did not reset."
diff --git a/tests/pdf_viewer/test_pdf_download.py b/tests/pdf_viewer/test_pdf_download.py
@@ -52,9 +52,9 @@ def test_pdf_download(
     pdf.expect(lambda _: os.path.exists(saved_pdf_location))
 
     # Verify if the file exists
-    assert os.path.exists(
-        saved_pdf_location
-    ), f"The file was not downloaded to {saved_pdf_location}."
+    assert os.path.exists(saved_pdf_location), (
+        f"The file was not downloaded to {saved_pdf_location}."
+    )
 
     print(
         f"Test passed: The file {file_name} has been downloaded and is present at {saved_pdf_location}."
diff --git a/tests/preferences/test_check_for_updates.py b/tests/preferences/test_check_for_updates.py
diff --git a/tests/preferences/test_never_remember_history.py b/tests/preferences/test_never_remember_history.py
diff --git a/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py b/tests/scrolling_panning_zooming/test_mouse_wheel_zoom.py
diff --git a/tests/scrolling_panning_zooming/test_zoom_from_menu_bar.py b/tests/scrolling_panning_zooming/test_zoom_from_menu_bar.py
diff --git a/tests/scrolling_panning_zooming/test_zoom_menu_correlation.py b/tests/scrolling_panning_zooming/test_zoom_menu_correlation.py
diff --git a/tests/security_and_privacy/test_private_session_history_exclusion.py b/tests/security_and_privacy/test_private_session_history_exclusion.py
diff --git a/tests/security_and_privacy/test_tls_v1_2_protocol.py b/tests/security_and_privacy/test_tls_v1_2_protocol.py
diff --git a/tests/security_and_privacy/test_trackers_crypto_fingerprint_blocked.py b/tests/security_and_privacy/test_trackers_crypto_fingerprint_blocked.py
diff --git a/tests/tabs/test_list_all_tabs.py b/tests/tabs/test_list_all_tabs.py

Original file line number	Diff line number	Diff line change
`@@ -89,4 +89,6 @@ def test_demo_ad_address_data_captured_in_doorhanger_and_stored(`
`89`	`89`	`found_address_data = any(`
`90`	`90`	`all(value in element.text for value in expected_values) for element in elements`
`91`	`91`	`)`
`92`		`- assert found_address_data, "Street, city, state (if applicable), zip, or country were not found in any of the address entries!"`
	`92`	`+ assert found_address_data, (`
	`93`	`+ "Street, city, state (if applicable), zip, or country were not found in any of the address entries!"`
	`94`	`+ )`