0.5 prototype

Author: snps-kumykov

examples/client/file_hierarchy_report.py

@@ -30,6 +30,7 @@
 import argparse
 import logging
 import sys
+import io
 import os
 import re
 import time
@@ -40,10 +41,11 @@
 import ijson
 from blackduck import Client
 from zipfile import ZipFile
+from pprint import pprint
 
 program_description = \
 '''Generate version detail reports (source and components) and consolidate information on source matches, with license 
-and component matched.  Removes matches found underneith other matched components in the source tree (configurable).
+and component matched.  Removes matches found underneath other matched components in the source tree (configurable).
 
 This script assumes a project version exists and has scans associated with it (i.e. the project is not scanned as part of this process).
 
@@ -83,39 +85,21 @@ def log_config(debug):
     logging.getLogger("urllib3").setLevel(logging.WARNING)
     logging.getLogger("blackduck").setLevel(logging.WARNING)
 
-def parse_parameter():
-    parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter)
-    parser.add_argument("project",
-                        metavar="project",
-                        type=str,
-                        help="Provide the BlackDuck project name.")
-    parser.add_argument("version",
-                        metavar="version",
-                        type=str,
-                        help="Provide the BlackDuck project version name.")
-    parser.add_argument("-kh",
-                        "--keep_hierarchy",
-                        action='store_true',
-                        help="Set to keep all entries in the sources report. Will not remove components found under others.")
-    parser.add_argument("-rr",
-                        "--report_retries",
-                        metavar="",
-                        type=int,
-                        default=RETRY_LIMIT,
-                        help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.")
-    parser.add_argument("-t",
-                        "--timeout",
-                        metavar="",
-                        type=int,
-                        default=15,
-                        help="Timeout for REST-API. Some API may take longer than the default 15 seconds")
-    parser.add_argument("-r",
-                        "--retries",
-                        metavar="",
-                        type=int,
-                        default=3,
-                        help="Retries for REST-API. Some API may need more retries than the default 3 times")
-    return parser.parse_args()
+def find_project_by_name(bd, project_name):
+    params = {
+        'q': [f"name:{project_name}"]
+    }
+    projects = [p for p in bd.get_resource('projects', params=params) if p['name'] == project_name]
+    assert len(projects) == 1, f"Project {project_name} not found."
+    return projects[0]
+
+def find_project_version_by_name(bd, project, version_name):
+    params = {
+        'q': [f"versionName:{version_name}"]
+    }
+    versions = [v for v in bd.get_resource('versions', project, params=params) if v['versionName'] == version_name]
+    assert len(versions) == 1, f"Project version {version_name} for project {project['name']} not found"
+    return versions[0]
 
 def get_bd_project_data(hub_client, project_name, version_name):
     """ Get and return project ID, version ID. """
@@ -136,55 +120,53 @@ def get_bd_project_data(hub_client, project_name, version_name):
 
     return project_id, version_id
 
-def report_create(hub_client, url, body):
-    """ 
-    Request BlackDuck to create report. Requested report is included in the request payload.
-    """
-    res = hub_client.session.post(url, headers={'Content-Type': BLACKDUCK_REPORT_MEDIATYPE}, json=body)
-    if res.status_code != 201:
-        sys.exit(f"BlackDuck report creation failed with status {res.status_code}!") 
-    return res.headers['Location'] # return report_url
-
-def report_download(hub_client, report_url, project_id, version_id, retries):
-    """
-    Download the generated report after the report completion. We will retry until reaching the retry-limit.
-    """
-    while retries:
-        res = hub_client.session.get(report_url, headers={'Accept': BLACKDUCK_REPORT_MEDIATYPE})
-        if res.status_code == 200 and (json.loads(res.content))['status'] == "COMPLETED":
-            report_id = report_url.split("reports/", 1)[1]
-            download_url = (((blackduck_report_download_api.replace("{projectId}", project_id))
-                     .replace("{projectVersionId}", version_id))
-                     .replace("{reportId}", report_id))  
-            res = hub_client.session.get(download_url, 
-                                         headers={'Content-Type': 'application/zip', 'Accept':'application/zip'})
-            if res.status_code != 200:
-                sys.exit(f"BlackDuck report download failed with status {res.status_code} for {download_url}!")
-            return res.content
-        elif res.status_code != 200:
-            sys.exit(f"BlackDuck report creation not completed successfully with status {res.status_code}")
-        else:
-            retries -= 1
-            logging.info(f"Waiting for the report generation for {report_url} with the remaining retries {retries} times.")
-            time.sleep(RETRY_TIMER)
-    sys.exit(f"BlackDuck report for {report_url} was not generated after retries {RETRY_TIMER} sec * {retries} times!")
-
-def get_version_detail_report(hub_client, project_id, version_id, retries):
-    """ Create and get BOM component and BOM source file report in json. """
-    create_version_url = blackduck_create_version_report_api.replace("{projectVersionId}", version_id)
-    body = {
+def create_version_details_report(bd, version):
+    version_reports_url = bd.list_resources(version).get('versionReport')
+    post_data = {
         'reportFormat' : 'JSON',
         'locale' : 'en_US',
-        'versionId' : f'{version_id}',
+        'versionId': version['_meta']['href'].split("/")[-1],
         'categories' : [ 'COMPONENTS', 'FILES' ] # Generating "project version" report including components and files
     }
-    report_url = report_create(hub_client, create_version_url, body)
-    # Zipped report content is received and write the content to a local zip file 
-    content = report_download(hub_client, report_url, project_id, version_id, retries)
-    output_file = blackduck_version_report_filename.replace("{projectVersionId}", version_id)
-    with open(output_file, "wb") as f:
-        f.write(content)
-        return output_file
+
+    bd.session.headers["Content-Type"] = "application/vnd.blackducksoftware.report-4+json"
+    r = bd.session.post(version_reports_url, json=post_data)
+    if (r.status_code == 403):
+        logging.debug("Authorization Error - Please ensure the token you are using has write permissions!")
+    r.raise_for_status()
+    pprint(r.headers)
+    location = r.headers.get('Location')
+    assert location, "Hmm, this does not make sense. If we successfully created a report then there needs to be a location where we can get it from"
+    return location
+
+def download_report(bd, location, retries):
+    report_id = location.split("/")[-1]
+    print (location)
+    url_data = location.split('/')
+    url_data.pop(4)
+    url_data.pop(4)
+    download_link = '/'.join(url_data)
+    print(download_link)
+    if retries:
+        logging.debug(f"Retrieving generated report from {location}")
+        response = bd.session.get(location)
+        report_status = response.json().get('status', 'Not Ready')
+        if response.status_code == 200 and report_status == 'COMPLETED':
+            response = bd.session.get(download_link, headers={'Content-Type': 'application/zip', 'Accept':'application/zip'})
+            pprint(response)
+            if response.status_code == 200:
+                return response.content
+            else:
+                logging.error("Ruh-roh, not sure what happened here")
+                return None
+        else:
+            logging.debug(f"Report status request {response.status_code} {report_status} ,waiting {retries} seconds then retrying...")
+            time.sleep(60)
+            retries -= 1
+            return download_report(bd, location, retries)
+    else:
+        logging.debug(f"Failed to retrieve report {report_id} after multiple retries")
+        return None
 
 def get_blackduck_version(hub_client):
     url = hub_client.base_url + BLACKDUCK_VERSION_API
@@ -194,68 +176,46 @@ def get_blackduck_version(hub_client):
     else:
         sys.exit(f"Get BlackDuck version failed with status {res.status_code}")
 
-def generate_file_report(hub_client, project_id, version_id, keep_hierarchy, retries):
-    """ 
-    Create a consolidated file report from BlackDuck project version source and components reports.
-    Remarks: 
-    """
-    if not os.path.exists(REPORT_DIR):
-        os.makedirs(REPORT_DIR)
-    
-    # Report body - Component BOM, file BOM with Discoveries data
-    version_report_zip = get_version_detail_report(hub_client, project_id, version_id, retries)
-    with ZipFile(f"./{version_report_zip}", "r") as vzf:
-        vzf.extractall()
-    for i, unzipped_version in enumerate(vzf.namelist()):
-        if re.search(r"\bversion.+json\b", unzipped_version) is not None:
-            break
-        if i + 1 >= len(vzf.namelist()):
-            sys.exit(f"Version detail file not found in the downloaded report: {version_report_zip}!")
-    
-    # Report body - Component BOM report
-    with open(f"./{unzipped_version}", "r") as uvf:
-        for i, comp_bom in enumerate(ijson.items(uvf, 'aggregateBomViewEntries.item')):
-            logging.info(f"{comp_bom['componentName']}")
-        logging.info(f"Number of the reported components {i+1}")
-      
+def parse_command_args():
+    parser = argparse.ArgumentParser(description=program_description, formatter_class=argparse.RawTextHelpFormatter)
+    parser.add_argument("-u", "--base-url",     required=True, help="Hub server URL e.g. https://your.blackduck.url")
+    parser.add_argument("-t", "--token-file",   required=True, help="File containing access token")
+    parser.add_argument("-nv", "--no-verify",   action='store_false', help="Disable TLS certificate verification")
+    parser.add_argument("-d", "--debug", action='store_true', help="Set debug output on")
+    parser.add_argument("-pn", "--project-name", required=True, help="Project Name")
+    parser.add_argument("-pv", "--project-version-name", required=True, help="Project Version Name")
+    parser.add_argument("-kh", "--keep_hierarchy", action='store_true', help="Set to keep all entries in the sources report. Will not remove components found under others.")
+    parser.add_argument("--report-retries", metavar="", type=int, default=RETRY_LIMIT, help="Retries for receiving the generated BlackDuck report. Generating copyright report tends to take longer minutes.")
+    parser.add_argument("--timeout", metavar="", type=int, default=60, help="Timeout for REST-API. Some API may take longer than the default 60 seconds")
+    parser.add_argument("--retries", metavar="", type=int, default=4, help="Retries for REST-API. Some API may need more retries than the default 4 times")
+    return parser.parse_args()
 
 def main():
-    args = parse_parameter()
-    debug = 0
+    args = parse_command_args()
+    with open(args.token_file, 'r') as tf:
+        token = tf.readline().strip()
     try:
-        if args.project == "":
-            sys.exit("Please set BlackDuck project name!")
-        if args.version == "":
-            sys.exit("Please set BlackDuck project version name!")
-
-        with open(".restconfig.json", "r") as f:
-            config = json.load(f)
-            # Remove last slash if there is, otherwise REST API may fail.
-            if re.search(r".+/$", config['baseurl']):
-                bd_url = config['baseurl'][:-1]
-            else:
-                bd_url = config['baseurl']
-            bd_token = config['api_token']
-            bd_insecure = not config['insecure']
-            if config['debug']:
-                debug = 1
-        
-        log_config(debug)    
-
-        hub_client = Client(token=bd_token,
-                            base_url=bd_url,
-                            verify=bd_insecure,
+        log_config(args.debug)    
+        hub_client = Client(token=token,
+                            base_url=args.base_url,
+                            verify=args.no_verify,
                             timeout=args.timeout,
                             retries=args.retries)
 
-        project_id, version_id = get_bd_project_data(hub_client, args.project, args.version)
+        project = find_project_by_name(hub_client, args.project_name)
+        version = find_project_version_by_name(hub_client, project, args.project_version_name)
+        pprint(version)
+        location = create_version_details_report(hub_client, version)
+        pprint(location)
+        report_zip = download_report(hub_client, location, args.report_retries)
+        pprint(report_zip)
+        logging.debug(f"Deleting report from Black Duck {hub_client.session.delete(location)}")
+        zip=ZipFile(io.BytesIO(report_zip), "r")
+        pprint(zip.namelist())
+        report_data = {name: zip.read(name) for name in zip.namelist()}
+        filename = [i for i in report_data.keys() if i.endswith(".json")][0]
+        pprint(json.loads(report_data[filename]))
 
-        generate_file_report(hub_client,
-                             project_id,
-                             version_id,
-                             args.keep_hierarchy,
-                             args.report_retries
-                             )
 
     except (Exception, BaseException) as err:
         logging.error(f"Exception by {str(err)}. See the stack trace")