Merge pull request #88 from KeerthanaAP/rc_builds

Added support to get job runs details based on the release

Author: ppc64le-cloud-bot

CI_JobHistory.py

@@ -2,7 +2,7 @@
 import urllib3
 from tabulate import tabulate
 import re
-from datetime import datetime
+from datetime import datetime,timedelta
 import monitor
 import argparse
 import configparser
@@ -15,6 +15,36 @@
 
 JENKINS = config_vars.get('Settings', 'JENKINS')
 
+def get_release_date():
+    '''
+    Fetch release start date and end date
+    Returns:
+        Date: Start date
+        Date: End date
+    '''
+    release = input("Enter the release: ") #release that needs to be checked
+    assert len(release.strip()) != 0, "Release value should not be empty"
+    nxt_rls = input("Enter the next release (provide latest if no next release): ") #next release or latest
+    assert len(nxt_rls.strip()) != 0 , "Next release value should not be empty"
+    r = re.compile(r"^\d*[.,]?\d*$")
+    if re.match(r,release)==False:
+        print("Enter valid release version")
+
+    try:
+
+        release_date_str = monitor.fetch_release_date(release)
+        release_date = datetime.strptime(release_date_str,"%Y-%m-%d  %H:%M:%S")
+
+        if nxt_rls != "latest":
+            nxt_rls_str = monitor.fetch_release_date(nxt_rls)
+            nxt_rls_date = datetime.strptime(nxt_rls_str,"%Y-%m-%d  %H:%M:%S")
+        else:
+            nxt_rls_date=datetime.now()
+    except ValueError:
+        print("Invalid date format")
+        return None
+
+    return nxt_rls_date,release_date
 
 
 def get_date_input():
@@ -33,18 +63,14 @@ def get_date_input():
     elif JENKINS == "True":
         date_str_1 = config_vars.get('Settings', 'before_date')
         date_str_2 = config_vars.get('Settings', 'after_date')
-
     try:
         start_date = datetime.strptime(date_str_1,"%Y-%m-%d")
         end_date = datetime.strptime(date_str_2,"%Y-%m-%d")
-        start_date = start_date.date()
-        end_date = end_date.date()
         return start_date,end_date
     except ValueError:
         print("Invalid date format")
         return None
 
-
 def check_for_node_crashes(job_list, zone):
     """
     Check for node crash across all the provided job list
@@ -218,6 +244,7 @@ def get_query_options():
         print("4. Failed testcases")
         print("5. Get builds with testcase failure")
         print("6. Get testcase failure frequency")
+        print("7. Get build based on release ")
         option = input("Enter the option: ")
 
     elif JENKINS == "True":
@@ -264,9 +291,14 @@ def main():
 
     ci_list = display_ci_links(config_data,filter)
     if isinstance(ci_list,dict):
-        start_date,end_date = get_date_input()
+        option = get_query_options()
+
+        if option == '7':
+            start_date,end_date = get_release_date()
+        else:
+            start_date,end_date = get_date_input()
+
         if start_date != None and end_date != None:
-            option = get_query_options()
             print("Checking runs from",end_date,"to",start_date)
 
             if option == '1':
@@ -337,6 +369,11 @@ def main():
                     print(len(spy_links),"builds have run in the date range of",start_date,"to",end_date)
                     print_tc_frequency(spy_links,zone=args.zone,tc_name=tc_list)
                     monitor.final_job_list = []
+            if option == '7':
+                for ci_name,ci_link in ci_list.items():
+                    spy_links = monitor.get_jobs_with_date(ci_link,start_date,end_date)
+                    monitor.get_detailed_job_info(spy_links,ci_name,zone=args.zone)
+                    monitor.final_job_list = []
 
 if __name__ == "__main__":
     main()

aggregate.py

@@ -84,22 +84,22 @@ def get_job_name():
 def get_builds_with_same_nightly(job_name,nightly_image):
     builds=[]
     agg_builds = []
-    pattern = r'\d{4}-\d{2}-\d{2}'
+    pattern = r'\d{4}-\d{2}-\d{2}-\d{6}'
     match = re.search(pattern,nightly_image)
 
     if match != None:
         date_str = match.group()
-        nightly_date = datetime.strptime(date_str,"%Y-%m-%d").date()
-        current_date = datetime.now().date()
+        nightly_date = datetime.strptime(date_str,"%Y-%m-%d-%H%M%S")
+        current_date = datetime.now()
         builds=monitor.get_jobs_with_date(job_name,current_date,nightly_date)
         for spylink in reversed(builds):
             ng = ""
             _, ng = monitor.get_quota_and_nightly(spylink)
-            pattern_1 = r'\d{4}-\d{2}-\d{2}'
+            pattern_1 = r'\d{4}-\d{2}-\d{2}-\d{6}'
             match_1 = re.search(pattern_1,ng)
             if match_1 != None:
                 date_str_1 = match_1.group()
-                ng_date = datetime.strptime(date_str_1,"%Y-%m-%d").date()
+                ng_date = datetime.strptime(date_str_1,"%Y-%m-%d-%H%M%S")
                 if ng_date > nightly_date:
                     break
                 else:

monitor.py

@@ -11,6 +11,33 @@
 PROW_URL = ""
 PROW_VIEW_URL = "https://gcsweb-ci.apps.ci.l2s4.p1.openshiftapps.com/gcs"
 final_job_list=[]
+RELEASE_URL = "https://ppc64le.ocp.releases.ci.openshift.org/releasestream/4-stable-ppc64le/release/"
+
+def fetch_release_date(release):
+    '''
+    Returns the created date of release
+    '''
+    url = RELEASE_URL + release
+    try:
+        response = requests.get(url, verify=False, timeout=15)
+        if response.status_code == 200:
+             soup = BeautifulSoup(response.text, 'html.parser')
+             p_elements = soup.find_all("p")
+             for p in p_elements:
+                p_ele = p.string
+                if p_ele:
+                    if "Created:" in p_ele:
+                        start_date = p_ele.split(" ")[1]+" "+p_ele.split(" ")[2]
+                        break
+             return start_date
+        else:
+            return "failed to get the release page"
+    except requests.Timeout as e:
+        return "Request timed out"
+    except requests.RequestException as e:
+        return "Error while sending request to url"
+    except json.JSONDecodeError as e:
+        return "Failed to extract the spy-links"
 
 def set_prow_url(ci_job_type: str)->str:
     '''
@@ -47,7 +74,7 @@ def load_config(config_file):
         sys.exit(1)
 
 def get_current_date():
-    return datetime.now().date()
+    return datetime.now()
 
 def parse_job_date(date):
 
@@ -62,8 +89,7 @@ def parse_job_date(date):
     '''
 
     parse_date=datetime.strptime(date,"%Y-%m-%dT%H:%M:%SZ")
-    job_run_date=parse_date.date()
-    return job_run_date
+    return parse_date
 
 
 def get_jobs(prow_link):