Get builds based on nightly image

Signed-off-by: SurajGudaji <[email protected]>

Author: SurajGudaji

CI_DailyBuildUpdates.py

@@ -20,18 +20,20 @@ def main():
     elif args.job_type == 'z':
         config_file = 'z_periodic.json'
     elif args.job_type == 'pa':
-        config_file = 'p_auxillary.json'
+        config_file = 'p_auxillary.json' 
 
     monitor.PROW_URL = monitor.set_prow_url(args.job_type)
     config_data = monitor.load_config(config_file)
     if args.info_type == "brief":
         summary_list = []
         for ci_name,ci_link in config_data.items():
-            summary_list.extend(monitor.get_brief_job_info(ci_name,ci_link,zone=args.zone))
+            build_list = monitor.get_jobs(ci_link)
+            summary_list.extend(monitor.get_brief_job_info(build_list,ci_name,zone=args.zone))
         print(tabulate(summary_list, headers='keys', tablefmt="pipe", stralign='left'))
     elif args.info_type == "detailed":
         for ci_name,ci_link in config_data.items():
-            monitor.get_detailed_job_info(ci_name,ci_link,zone=args.zone)
+            build_list = monitor.get_jobs(ci_link)
+            monitor.get_detailed_job_info(build_list,ci_name,zone=args.zone)
 
 if __name__ == "__main__":
     main()

CI_JobHistory.py

@@ -235,6 +235,8 @@ def get_testcase_names():
             tc_list =  tc_name.split(",")
             return tc_list
 
+
+
 def main():
     parser = argparse.ArgumentParser(description='Get the job history')
     parser.add_argument('--zone', help='specify the lease/zone', type= lambda arg:arg.split(','))
@@ -273,13 +275,15 @@ def main():
             if option == '2':
                 summary_list = []
                 for ci_name,ci_link in ci_list.items():
-                    summary_list.extend(monitor.get_brief_job_info(ci_name,ci_link,start_date,end_date,zone=args.zone))
+                    spy_links = monitor.get_jobs_with_date(ci_link,start_date,end_date)
+                    summary_list.extend(monitor.get_brief_job_info(spy_links,ci_name,zone=args.zone))
                     monitor.final_job_list = []
                 print(tabulate(summary_list, headers='keys', tablefmt="pipe", stralign='left'))
 
             if option == '3':
                 for ci_name,ci_link in ci_list.items():
-                    monitor.get_detailed_job_info(ci_name,ci_link,start_date,end_date,zone=args.zone)
+                    spy_links = monitor.get_jobs_with_date(ci_link,start_date,end_date)
+                    monitor.get_detailed_job_info(spy_links,ci_name,zone=args.zone)
                     monitor.final_job_list = []
 
             if option == '4':

README.md

@@ -78,3 +78,10 @@ Create a virtualenv if required and install required packages using "pip install
     1. builds: This argument accepts int value, which will query for failed testcases in "n" latest build run in the CI, default value set is 10.
     2. frequency: This argument accepts int value, which specifies the frequency threshold and report the testcases which are failing above the frequency, default value set is 3.
 
+
+4. **aggregate.py:** The aggregate.py script gets detailed information of all the builds which have run using the provided nightly image. 
+
+    ```python3 tracker.py``` This script requires the following input
+
+    1. selected_ci: Jobs from where to fetch the builds.
+    2. nightly: Name of the Release image.

aggregate.py

@@ -0,0 +1,124 @@
+from tabulate import tabulate
+import re
+from datetime import datetime
+import monitor
+import configparser
+import argparse
+
+config_vars = configparser.ConfigParser()
+config_vars.read('config.ini')
+
+JENKINS = config_vars.get('Settings', 'JENKINS')
+
+def get_nightly_name():
+    if JENKINS == "False":
+        nightly = input("Enter the name of nightly image: ")
+        return nightly
+    elif JENKINS == "True":
+        nightly = config_vars.get('Settings', 'nightly')
+        return nightly
+
+def get_job_name():
+
+    '''
+    Gets selected CI input.
+
+    Returns:
+        dict: Dictionary of selected job name and job link 
+    '''
+
+    parser = argparse.ArgumentParser(description='Get the job history')
+    parser.add_argument('--job_type', default='p', choices=['p','z','pa'], help= 'Specify the CI job type (Power(p) or s390x(z) or Power Auxillary(pa)), default is p')
+
+    args = parser.parse_args()
+
+    if args.job_type == 'p':
+        config_file = 'p_periodic.json'
+    elif args.job_type == 'z':
+        config_file = 'z_periodic.json'
+    elif args.job_type == 'pa':
+        config_file = 'p_auxillary.json'
+    
+    monitor.PROW_URL = monitor.set_prow_url(args.job_type)
+    config_data = monitor.load_config(config_file)
+
+    j=0
+    ci_name_list = []
+    options_int_list = []
+    selected_config_data = {}
+
+    if JENKINS == "False":
+        for ci_name in config_data.keys():
+            j=j+1
+            ci_name_list.append(ci_name)
+            print(j,'',ci_name)
+        option = input("Select the required ci's serial number with a space ")
+        selected_options = option.split()
+        for ci in selected_options:
+            try:
+                ci_to_int = int(ci)
+                if 0 < ci_to_int <= len(config_data):
+                    options_int_list.append(ci_to_int)
+                else:
+                    return_value = "Enter the options in range of 1 to " + str(len(config_data)+1)
+                    print(return_value)
+                    return "ERROR"
+            except ValueError:
+                return "Enter valid options"
+    elif JENKINS == "True":
+        for ci_name in config_data.keys():
+            j=j+1
+            ci_name_list.append(ci_name)
+   
+        selected_ci = config_vars.get('Settings', 'selected_ci')
+        options_int_list = [int(value) for value in selected_ci.split(',')]
+
+    for i in options_int_list:
+        config_temp_data = {ci_name_list[i-1]: config_data[ci_name_list[i-1]]}
+        selected_config_data.update(config_temp_data)
+        config_temp_data = {}
+
+    return selected_config_data
+
+
+def get_builds_with_same_nightly(job_name,nightly_image):
+    builds=[]
+    agg_builds = []
+    pattern = r'\d{4}-\d{2}-\d{2}'
+    match = re.search(pattern,nightly_image)
+    
+    if match != None:
+        date_str = match.group()
+        nightly_date = datetime.strptime(date_str,"%Y-%m-%d").date()
+        current_date = datetime.now().date()
+        builds=monitor.get_jobs_with_date(job_name,current_date,nightly_date)
+        for spylink in reversed(builds):
+            ng = ""
+            _, ng = monitor.get_quota_and_nightly(spylink)
+            pattern_1 = r'\d{4}-\d{2}-\d{2}'
+            match_1 = re.search(pattern_1,ng)
+            if match_1 != None:
+                date_str_1 = match_1.group()
+                ng_date = datetime.strptime(date_str_1,"%Y-%m-%d").date()
+                if ng_date > nightly_date:
+                    break
+                else:
+                    if nightly_image in ng:
+                        agg_builds.append(spylink)
+            else:
+                continue
+        return agg_builds
+
+def main():
+    nightly_image = get_nightly_name()
+    selected_jobs = get_job_name()
+    print("****************************************")
+    print("Payload: ",nightly_image)
+    print("****************************************")
+    for job_name,job_link in selected_jobs.items():
+        build_list = []
+        build_list = get_builds_with_same_nightly(job_link,nightly_image)
+        monitor.get_detailed_job_info(build_list,job_name)
+
+if __name__ == "__main__":
+    main()

config.ini

@@ -5,3 +5,4 @@ before_date = __BDATE__
 after_date = __ADATE__
 query_option = __QP__
 tc_name = __TC__
+nightly = __NIGHTLY__