Merge pull request #2707 from cms-sw/merge-prs-cache

collect merged details for easy access via IB dashboard

Author: smuzaffar

github_utils.py

@@ -293,6 +293,17 @@ def cache_invalid_pr(pr_id, cache):
     cache["dirty"] = True
 
 
+def get_pr_data(repo_name, pr_number, cmsprs):
+    pr_md5 = md5((pr_number + "\n").encode()).hexdigest()
+    pr_cache = join(cmsprs, repo_name, pr_md5[0:2], pr_md5[2:] + ".json")
+    print("Checking cached file: " + pr_cache)
+    if exists(pr_cache):
+        with open(pr_cache) as ref:
+            return json.load(ref)
+    print("  Pull request cache %s for %s#%s not found." % (pr_cache, repo_name, pr_number))
+    return {}
+
+
 def fill_notes_description(notes, repo_name, cmsprs, cache={}):
     new_notes = {}
     for log_line in notes.splitlines():
@@ -310,14 +321,10 @@ def fill_notes_description(notes, repo_name, cmsprs, cache={}):
             continue
         print("Checking ", pr_number, author, parent_hash)
         try:
-            pr_md5 = md5((pr_number + "\n").encode()).hexdigest()
-            pr_cache = join(cmsprs, repo_name, pr_md5[0:2], pr_md5[2:] + ".json")
-            print("Checking cached file: " + pr_cache)
-            if not exists(pr_cache):
-                print("  Cache does not exists: ", pr_cache)
+            pr = get_pr_data(repo_name, pr_number, cmsprs)
+            if not pr:
                 cache_invalid_pr(pr_hash_id, cache)
                 continue
-            pr = json.load(open(pr_cache))
             if not "auther_sha" in pr:
                 print("  Invalid/Indirect PR", pr)
                 cache_invalid_pr(pr_hash_id, cache)

report-summary-merged-prs.py

@@ -16,7 +16,7 @@
 from pprint import pformat
 
 from cmsutils import get_config_map_properties
-from github_utils import get_merge_prs
+from github_utils import get_merge_prs, get_pr_data
 from cms_static import GH_CMSSW_REPO, GH_CMSSW_ORGANIZATION
 from releases import CMSSW_DEVEL_BRANCH
 from socket import setdefaulttimeout
@@ -2043,6 +2043,31 @@ class PossibleUnitTestResults(object):
     structure = identify_release_groups(results)
     fix_results(results)
 
+    merged_pr_cache = {}
+    for rx in results:
+        for res in rx["comparisons"]:
+            if not res.get("release_queue", False):
+                continue
+            rel_que = "_".join(res.get("release_queue").split("_")[:3])
+            if not rel_que in merged_pr_cache:
+                merged_pr_cache[rel_que] = {"cmsdist": {}, "cmssw": {}}
+            prs = res.get("merged_prs", [])
+            for pr in prs:
+                merged_pr_cache[rel_que]["cmssw"][pr["number"]] = get_pr_data(
+                    "cms-sw/cmssw", pr["number"], CMS_PRS
+                )
+            prs = res.get("cmsdist_merged_prs", {})
+            for arch in prs:
+                for pr in prs[arch]:
+                    merged_pr_cache[rel_que]["cmsdist"][pr["number"]] = get_pr_data(
+                        "cms-sw/cmsdist", pr["number"], CMS_PRS
+                    )
+
+    for rel_que in merged_pr_cache:
+        out_json = open("prs/%s.json" % rel_que, "w")
+        json.dump(merged_pr_cache[rel_que], out_json, sort_keys=True, indent=4)
+        out_json.close()
+
     prod_archs = get_production_archs(get_config_map_properties())
     prod_ib_index = {}
     for rx in results: