Merge pull request #80 from blackducksoftware/gsnyder/issue-79-re-generating-notices-report-in-html-format

fixing issue 79 regarding generating notices report in JSON format

Author: Glenn Snyder

blackduck/HubRestApi.py

@@ -424,7 +424,7 @@ def _get_policy_url(self):
 
     def get_policies(self, parameters={}):
         url = self._get_policy_url() + self._get_parameter_string(parameters)
-        headers = {'Accept': 'application/vnd.blackducksoftware.policy-4+json'}
+        headers = {'Accept': 'application/json'}
         response = self.execute_get(url, custom_headers=headers)
         return response.json()
 
@@ -540,12 +540,12 @@ def create_version_reports(self, version, report_list, format="CSV"):
         version_reports_url = self.get_link(version, 'versionReport')
         return self.execute_post(version_reports_url, post_data)
 
-    valid_notices_formats = ["TEXT", "HTML"]
+    valid_notices_formats = ["TEXT", "JSON"]
     def create_version_notices_report(self, version, format="TEXT"):
         assert format in HubInstance.valid_notices_formats, "Format must be one of {}".format(HubInstance.valid_notices_formats)
 
         post_data = {
-            'categories': HubInstance.valid_categories,
+            'categories': ["COPYRIGHT_TEXT"],
             'versionId': version['_meta']['href'].split("/")[-1],
             'reportType': 'VERSION_LICENSE',
             'reportFormat': format
@@ -554,9 +554,28 @@ def create_version_notices_report(self, version, format="TEXT"):
         return self.execute_post(notices_report_url, post_data)
 
     def download_report(self, report_id):
+        # TODO: Fix me, looks like the reports should be downloaded from different paths than the one here, and depending on the type and format desired the path can change
         url = self.get_urlbase() + "/api/reports/{}".format(report_id)
         return self.execute_get(url, {'Content-Type': 'application/zip', 'Accept':'application/zip'})
 
+    def download_notification_report(self, report_location_url):
+        '''Download the notices report using the report URL. Inspect the report object to determine
+        the format and use the appropriate media header'''
+        custom_headers = {'Accept': 'application/vnd.blackducksoftware.report-4+json'}
+        response = self.execute_get(report_location_url, custom_headers=custom_headers)
+        report_obj = response.json()
+
+        if report_obj['reportFormat'] == 'TEXT':
+            download_url = self.get_link(report_obj, "download") + ".json"
+            logging.debug("downloading report from {}".format(download_url))
+            response = self.execute_get(download_url, {'Accept': 'application/zip'})
+        else:
+            # JSON
+            contents_url = self.get_link(report_obj, "content")
+            logging.debug("retrieving report contents from {}".format(contents_url))
+            response = self.execute_get(contents_url, {'Accept': 'application/json'})
+        return response, report_obj['reportFormat']
+
     ##
     #
     # (Global) Vulnerability reports

examples/generate_notices_report_for_project_version.py

@@ -10,42 +10,51 @@
 
 import argparse
 import json
+import logging
+import sys
 import time
 import zipfile
 
 parser = argparse.ArgumentParser("A program to generate the notices file for a given project-version")
 parser.add_argument("project_name")
 parser.add_argument("version_name")
-parser.add_argument("--zip_file_name", default="notices_report.zip")
-parser.add_argument("--reports",
-	default="version,scans,components,vulnerabilities,source", 
-	help="Comma separated list (no spaces) of the reports to generate - version, scans, components, vulnerabilities, source, and cryptography reports (default: all, except cryptography")
-parser.add_argument('--format', default='TEXT', choices=["HTML", "TEXT"], help="Report format - choices are TEXT or HTML")
+parser.add_argument('-f', "--file_name_base", default="notices_report", help="Base file name to write the report data into. If the report format is TEXT a .zip file will be created, otherwise a .json file")
+parser.add_argument('-r', '--report_format', default='TEXT', choices=["JSON", "TEXT"], help="Report format - choices are TEXT or HTML")
 
 args = parser.parse_args()
 
 hub = HubInstance()
 
-# TODO: Promote this to the API?
+logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG)
+
 class FailedReportDownload(Exception):
 	pass
 
-def download_report(location, filename, retries=4):
+def download_report(location, file_name_base, retries=10):
 	report_id = location.split("/")[-1]
 
 	if retries:
-		print("Retrieving generated report from {}".format(location))
-		response = hub.download_report(report_id)
+		logging.debug("Retrieving generated report from {}".format(location))
+		# response = hub.download_report(report_id)
+		response, report_format = hub.download_notification_report(location)
 		if response.status_code == 200:
-			with open(filename, "wb") as f:
-				f.write(response.content)
-			print("Successfully downloaded zip file to {} for report {}".format(filename, report_id))
+			if report_format == "TEXT":
+				filename = file_name_base + ".zip"
+				with open(filename, "wb") as f:
+					f.write(response.content)
+			else:
+				# JSON format
+				filename = file_name_base + ".json"
+				with open(filename, "w") as f:
+					json.dump(response.json(), f, indent=3)
+			logging.info("Successfully downloaded json file to {} for report {}".format(
+					filename, report_id))
 		else:
-			print("Failed to retrieve report {}".format(report_id))
-			print("Probably not ready yet, waiting 5 seconds then retrying...")
+			logging.warning("Failed to retrieve report {}".format(report_id))
+			logging.warning("Probably not ready yet, waiting 5 seconds then retrying (remaining retries={}".format(retries))
 			time.sleep(5)
 			retries -= 1
-			download_report(location, filename, retries)
+			download_report(location, file_name_base, retries)
 	else:
 		raise FailedReportDownload("Failed to retrieve report {} after multiple retries".format(report_id))
 
@@ -54,22 +63,23 @@ def download_report(location, filename, retries=4):
 if project:
 	version = hub.get_version_by_name(project, args.version_name)
 
-	response = hub.create_version_notices_report(version, args.format)
+	response = hub.create_version_notices_report(version, args.report_format)
 
 	if response.status_code == 201:
-		print("Successfully created reports ({}) for project {} and version {}".format(
-			args.reports, args.project_name, args.version_name))
+		logging.info("Successfully created notices report in {} format for project {} and version {}".format(
+			args.report_format, args.project_name, args.version_name))
 		location = response.headers['Location']
-		download_report(location, args.zip_file_name)
+		download_report(location, args.file_name_base)
+
 
 		# Showing how you can interact with the downloaded zip and where to find the
 		# output content. Uncomment the lines below to see how it works.
 
-		# with zipfile.ZipFile(zip_file_name, 'r') as zipf:
+		# with zipfile.ZipFile(zip_file_name_base, 'r') as zipf:
 		# 	with zipf.open("{}/{}/version-license.txt".format(args.project_name, args.version_name), "r") as license_file:
 		# 		print(license_file.read())
 	else:
-		print("Failed to create reports for project {} version {}, status code returned {}".format(
+		logging.error("Failed to create reports for project {} version {}, status code returned {}".format(
 			args.project_name, args.version_name, response.status_code))
 else:
-	print("Did not find project with name {}".format(args.project_name))
+	logging.warning("Did not find project with name {}".format(args.project_name))

examples/get_bom_components.py

@@ -5,17 +5,18 @@
 import json
 import logging
 import sys
-import timestring
 
 from blackduck.HubRestApi import HubInstance
 
 parser = argparse.ArgumentParser("Retreive BOM component info for the given project and version")
 parser.add_argument("project_name")
 parser.add_argument("version")
 group = parser.add_mutually_exclusive_group()
+group.add_argument("-l", "--limit", default=10, help="Set limit on number of components to retrieve")
 group.add_argument("-u", "--unreviewed", action='store_true')
 group.add_argument("-r", "--reviewed", action='store_true')
 parser.add_argument("-v", "--vulnerabilities", action='store_true', help="Get the vulnerability info for each of the components")
+parser.add_argument("-c", "--custom_fields", action='store_true', help="Get the custom field info for each of the components")
 
 args = parser.parse_args()
 
@@ -31,7 +32,11 @@
 
 components_url = hub.get_link(version, "components")
 
-response = hub.execute_get(components_url)
+components_url += "?limit={}".format(args.limit)
+
+custom_headers = {'Accept': 'application/vnd.blackducksoftware.bill-of-materials-6+json'}
+
+response = hub.execute_get(components_url, custom_headers=custom_headers)
 if response.status_code == 200:
     components = response.json()
     components = components.get('items', [])
@@ -49,4 +54,14 @@
             if response.status_code == 200:
                 vulnerabilities = response.json().get('items', [])
             component['vulnerabilities'] = vulnerabilities
+
+    if args.custom_fields:
+        for component in components:
+            custom_fields_url = hub.get_link(component, "custom-fields")
+            response = hub.execute_get(custom_fields_url, custom_headers=custom_headers)
+            custom_fields = []
+            if response.status_code == 200:
+                custom_fields = response.json().get('items', [])
+            component['custom_fields'] = custom_fields
+
     print(json.dumps(components))