Merge remote-tracking branch 'origin' into Hani/cm-newegg-us-ad

Author: philimon-reset

.github/workflows/l10n.yml

@@ -138,12 +138,16 @@ jobs:
         uses: actions/setup-python@v5
         with:
           python-version: "3.11"
+      - name: Set Environment (Scheduled Beta)
+        if: ${{ inputs.job_to_run == 'L10N-Linux' }}
+        run: |
+          echo "TESTRAIL_REPORT='true'" >> "$GITHUB_ENV";
+          echo "Running report for most recent Beta on Linux";
       - name: Install dependencies
         id: setup
         env:
           MANUAL_DOWNLOAD_LINK: ${{ inputs.linux_tarball_link }}
         run: |
-          echo "MANUAL='true'" >> "$GITHUB_ENV";
           echo "Running l10n tests on supplied executable";
           sudo apt install gnome-screenshot;
           uname -m;
@@ -225,7 +229,7 @@ jobs:
           New-item -Name .env -Value $env_contents -ItemType File -Force
           Write-Host "Running l10n tests on supplied executable"
       - name: Set Environment (Scheduled Beta)
-        if: ${{ inputs.job_to_run == 'Smoke-Windows' }}
+        if: ${{ inputs.job_to_run == 'L10N-Windows' }}
         run: |
           $env_contents = @"
           TESTRAIL_REPORT='true'

l10n_CM/run_l10n.py

@@ -13,7 +13,6 @@
 valid_flags = {"--run-headless", "-n", "--reruns", "--fx-executable", "--ci"}
 flag_with_parameter = {"-n", "--reruns"}
 valid_region = {"US", "CA", "DE", "FR"}
-
 valid_sites = {
     "demo",
     "amazon",
@@ -26,9 +25,7 @@
     "decathlon",
     "vans",
     "ebay",
-    "newegg",
 }
-
 live_sites = []
 
 LOCALHOST = "127.0.0.1"

modules/testrail.py

@@ -391,6 +391,7 @@ def update_test_cases(
         testrail_suite_id,
         test_case_ids=[],
         status="passed",
+        elapsed=[],
     ):
         """Given a project id, a run id, and a suite id, for each case given a status,
         update the test objects with the correct status code"""
@@ -408,13 +409,19 @@ def update_test_cases(
                     testrail_project_id, testrail_suite_id
                 )
             ]
-        data = {
-            "results": [
-                {"case_id": test_case_id, "status_id": status_key.get(status)}
-                for test_case_id in test_case_ids
-            ]
-        }
-        return self._update_test_run_results(testrail_run_id, data)
+
+        results = []
+        for i in range(len(test_case_ids)):
+            results.append(
+                {
+                    "case_id": test_case_ids[i],
+                    "status_id": status_key.get(status),
+                    "elapsed": elapsed[i],
+                }
+            )
+        logging.warning(f"Going to update cases with payload: {results}")
+
+        return self._update_test_run_results(testrail_run_id, {"results": results})
 
     # Private Methods
 

modules/testrail_integration.py

@@ -246,21 +246,33 @@ def mark_results(testrail_session: TestRail, test_results):
                 for result in existing_results[run_id]
             }
             suite_id = test_results[category][run_id][0].get("suite_id")
-            all_test_cases = [
-                result.get("test_case") for result in test_results[category][run_id]
-            ]
+
+            all_test_cases = []
+            all_durations = []
+            for result in test_results[category][run_id]:
+                all_test_cases.append(result.get("test_case"))
+                all_durations.append(result.get("duration"))
 
             # Don't set passed tests to another status.
-            test_cases = [tc for tc in all_test_cases if current_results.get(tc) != 1]
+            test_cases_ids = []
+            durations = []
+            for i, test_case in enumerate(all_test_cases):
+                if current_results.get(test_case) != 1:
+                    test_cases_ids.append(test_case)
+                    durations.append(all_durations[i])
+            logging.warning(
+                f"Setting the following test cases in run {run_id} to {category}: {test_cases_ids}"
+            )
             logging.warning(
-                f"Setting the following test cases in run {run_id} to {category}: {test_cases}"
+                f"Setting the following test cases {test_cases_ids} to duration {durations}"
             )
             testrail_session.update_test_cases(
                 test_results.get("project_id"),
                 testrail_run_id=run_id,
                 testrail_suite_id=suite_id,
-                test_case_ids=test_cases,
+                test_case_ids=test_cases_ids,
                 status=category,
+                elapsed=durations,
             )
 
 
@@ -407,6 +419,7 @@ def organize_entries(testrail_session: TestRail, expected_plan: dict, suite_info
     cases_in_suite = suite_info.get("cases")
     cases_in_suite = [int(n) for n in cases_in_suite]
     results = suite_info.get("results")
+    durations = suite_info.get("durations")
     plan_title = expected_plan.get("name")
 
     suite_entries = [
@@ -502,16 +515,18 @@ def organize_entries(testrail_session: TestRail, expected_plan: dict, suite_info
         "skipped": {},
     }
 
-    for test_case, outcome in results.items():
-        logging.info(f"{test_case}: {outcome}")
+    for test_case in results.keys():
+        outcome = results[test_case]
+        duration = durations[test_case]
+        logging.info(f"{test_case}: {outcome} {duration}")
         if outcome == "rerun":
             logging.info("Rerun result...skipping...")
             continue
         category = next(status for status in passkey if outcome in passkey.get(status))
         if not test_results[category].get(run_id):
             test_results[category][run_id] = []
         test_results[category][run_id].append(
-            {"suite_id": suite_id, "test_case": test_case}
+            {"suite_id": suite_id, "test_case": test_case, "duration": f"{duration}s"}
         )
 
     return test_results
@@ -547,7 +562,9 @@ def collect_changes(testrail_session: TestRail, report):
 
     version_str = metadata.get("fx_version")
     plan_title = get_plan_title(version_str, channel)
-    logging.info(plan_title)
+    logging.warning(
+        f"Got plan title: {plan_title} from version {version_str} and channel {channel}"
+    )
     plan_match = PLAN_NAME_RE.match(plan_title)
     (_, major) = [plan_match[n] for n in range(1, 3)]
     config = metadata.get("machine_config")
@@ -628,6 +645,7 @@ def collect_changes(testrail_session: TestRail, report):
     last_suite_id = None
     last_description = None
     results_by_suite = {}
+    durations_by_suite = {}
     full_test_results = {}
     tests = [
         test
@@ -655,11 +673,22 @@ def collect_changes(testrail_session: TestRail, report):
         # Tests reported as rerun are a problem -- we need to know pass/fail
         if outcome == "rerun":
             outcome = test.get("call").get("outcome")
-        logging.info(f"TC: {test_case}: {outcome}")
+        duration = (
+            test["setup"]["duration"]
+            + test["call"]["duration"]
+            + test["teardown"]["duration"]
+        )
+        logging.info(f"TC: {test_case}: {outcome} using {duration}s ")
 
         if not results_by_suite.get(suite_id):
             results_by_suite[suite_id] = {}
+            durations_by_suite[suite_id] = {}
         results_by_suite[suite_id][test_case] = outcome
+        durations_by_suite[suite_id].setdefault(test_case, 0)
+        durations_by_suite[suite_id][test_case] = round(
+            durations_by_suite[suite_id][test_case] + duration, 2
+        )
+
         if suite_id != last_suite_id:
             # When we get the last test_case in a suite, add entry, run, results
             if last_suite_id:
@@ -674,6 +703,7 @@ def collect_changes(testrail_session: TestRail, report):
                     "config_id": config_id,
                     "cases": cases_in_suite,
                     "results": results_by_suite[last_suite_id],
+                    "durations": durations_by_suite[last_suite_id],
                 }
 
                 full_test_results = merge_results(
@@ -694,6 +724,7 @@ def collect_changes(testrail_session: TestRail, report):
         "config_id": config_id,
         "cases": cases_in_suite,
         "results": results_by_suite[last_suite_id],
+        "durations": durations_by_suite[last_suite_id],
     }
 
     logging.info(f"n run {last_suite_id}, {last_description}")
@@ -706,6 +737,8 @@ def collect_changes(testrail_session: TestRail, report):
     full_test_results = merge_results(
         full_test_results, organize_entries(testrail_session, expected_plan, suite_info)
     )
+    logging.warning(f"full test results: {full_test_results}")
+
     return full_test_results