change reportablity logic to depend on the split load

Author: philimon-reset

choose_l10n_ci_set.py

@@ -79,6 +79,21 @@ def save_mappings(selected_container):
         f.writelines(current_running_mappings)
 
 
+def select_l10n_mappings(beta_version):
+    """
+    Select the correct l10n mappings.
+
+    Args:
+        beta_version: the current beta version.
+    """
+    beta_split = (beta_version % 3) + 1
+    if os.path.exists(f"l10n_CM/beta_run_splits/l10n_split_{beta_split}.json"):
+        with open(f"l10n_CM/beta_run_splits/l10n_split_{beta_split}.json", "r") as f:
+            return json.load(f)
+    else:
+        return valid_l10n_mappings()
+
+
 if __name__ == "__main__":
     if os.path.exists(".env"):
         with open(".env") as fh:
@@ -106,12 +121,8 @@ def save_mappings(selected_container):
         # failsafe beta_version
         beta_version = 0
     # choose split number
-    beta_version = (beta_version % 3) + 1
-    l10n_mappings = valid_l10n_mappings()
+    l10n_mappings = select_l10n_mappings(beta_version)
     sample_mappings = {k: v for k, v in l10n_mappings.items() if k.startswith("demo")}
-    if os.path.exists(f"l10n_CM/beta_run_splits/l10n_split_{beta_version}.json"):
-        with open(f"l10n_CM/beta_run_splits/l10n_split_{beta_version}.json", "r") as f:
-            l10n_mappings = json.load(f)
     if os.environ.get("TESTRAIL_REPORT") or os.environ.get("MANUAL"):
         # Run all tests if this is a scheduled beta or a manual run
         save_mappings(l10n_mappings)

modules/testrail_integration.py

@@ -1,10 +1,11 @@
+import json
 import logging
 import os
 import re
 import subprocess
 import sys
 
-from choose_l10n_ci_set import distribute_mappings_evenly, valid_l10n_mappings
+from choose_l10n_ci_set import select_l10n_mappings
 from modules import taskcluster as tc
 from modules import testrail as tr
 from modules.testrail import TestRail
@@ -187,11 +188,9 @@ def reportable(platform_to_test=None):
 
     plan_entries = this_plan.get("entries")
     if os.environ.get("FX_L10N"):
-        report = True
         beta_version = int(minor_num.split("b")[-1])
-        distributed_mappings = distribute_mappings_evenly(
-            valid_l10n_mappings(), beta_version
-        )
+        distributed_mappings = select_l10n_mappings(beta_version)
+        expected_mappings = sum(map(lambda x: len(x), distributed_mappings.values()))
         covered_mappings = 0
         # keeping this logic to still see how many mappings are reported.
         for entry in plan_entries:
@@ -205,13 +204,11 @@ def reportable(platform_to_test=None):
                             and platform in run_platform
                         ):
                             covered_mappings += 1
-                            report = False
         logging.warning(
-            f"Potentially matching run found for {platform}, may be reportable. (Found {covered_mappings} site/region mappings reported.)"
+            f"Potentially matching run found for {platform}, may be reportable. (Found {covered_mappings} site/region mappings reported, expected {expected_mappings}.)"
         )
-        logging.warning(f"Run is reportable: {report}")
-        # Only report when there is a new beta and no other site/region mappings are reported.
-        return report
+        # Only report when there is a new beta without a reported plan or if the selected split is not completely reported.
+        return covered_mappings < expected_mappings
     else:
         covered_suites = 0
         for entry in plan_entries: