Merge branch 'master' into update-create-users

Author: ericyame

blackduck/HubRestApi.py

@@ -181,6 +181,7 @@ def get_headers(self):
             return {
                 'X-CSRF-TOKEN': self.csrf_token, 
                 'Authorization': 'Bearer {}'.format(self.token), 
+                'Accept': 'application/json',
                 'Content-Type': 'application/json'}
         else:
             if self.bd_major_version == "3":
@@ -424,7 +425,7 @@ def _get_policy_url(self):
 
     def get_policies(self, parameters={}):
         url = self._get_policy_url() + self._get_parameter_string(parameters)
-        headers = {'Accept': 'application/vnd.blackducksoftware.policy-4+json'}
+        headers = {'Accept': 'application/json'}
         response = self.execute_get(url, custom_headers=headers)
         return response.json()
 
@@ -540,12 +541,12 @@ def create_version_reports(self, version, report_list, format="CSV"):
         version_reports_url = self.get_link(version, 'versionReport')
         return self.execute_post(version_reports_url, post_data)
 
-    valid_notices_formats = ["TEXT", "HTML"]
+    valid_notices_formats = ["TEXT", "JSON"]
     def create_version_notices_report(self, version, format="TEXT"):
         assert format in HubInstance.valid_notices_formats, "Format must be one of {}".format(HubInstance.valid_notices_formats)
 
         post_data = {
-            'categories': HubInstance.valid_categories,
+            'categories': ["COPYRIGHT_TEXT"],
             'versionId': version['_meta']['href'].split("/")[-1],
             'reportType': 'VERSION_LICENSE',
             'reportFormat': format
@@ -554,9 +555,48 @@ def create_version_notices_report(self, version, format="TEXT"):
         return self.execute_post(notices_report_url, post_data)
 
     def download_report(self, report_id):
+        # TODO: Fix me, looks like the reports should be downloaded from different paths than the one here, and depending on the type and format desired the path can change
         url = self.get_urlbase() + "/api/reports/{}".format(report_id)
         return self.execute_get(url, {'Content-Type': 'application/zip', 'Accept':'application/zip'})
 
+    def download_notification_report(self, report_location_url):
+        '''Download the notices report using the report URL. Inspect the report object to determine
+        the format and use the appropriate media header'''
+        custom_headers = {'Accept': 'application/vnd.blackducksoftware.report-4+json'}
+        response = self.execute_get(report_location_url, custom_headers=custom_headers)
+        report_obj = response.json()
+
+        if report_obj['reportFormat'] == 'TEXT':
+            download_url = self.get_link(report_obj, "download") + ".json"
+            logging.debug("downloading report from {}".format(download_url))
+            response = self.execute_get(download_url, {'Accept': 'application/zip'})
+        else:
+            # JSON
+            contents_url = self.get_link(report_obj, "content")
+            logging.debug("retrieving report contents from {}".format(contents_url))
+            response = self.execute_get(contents_url, {'Accept': 'application/json'})
+        return response, report_obj['reportFormat']
+
+    ##
+    #
+    # (Global) Vulnerability reports
+    #
+    ##
+    valid_vuln_status_report_formats = ["CSV", "JSON"]
+    def create_vuln_status_report(self, format="CSV"):
+        assert format in HubInstance.valid_vuln_status_report_formats, "Format must be one of {}".format(HubInstance.valid_vuln_status_report_formats)
+
+        post_data = {
+            "reportFormat": format,
+            "locale": "en_US"
+        }
+        url = self.get_apibase() + "/vulnerability-status-reports"
+        custom_headers = {
+            'Content-Type': 'application/vnd.blackducksoftware.report-4+json',
+            'Accept': 'application/vnd.blackducksoftware.report-4+json'
+        }
+        return self.execute_post(url, custom_headers=custom_headers, data=post_data)
+
     ##
     #
     # License stuff

blackduck/__version__.py

@@ -1,3 +1,3 @@
-VERSION = (0, 0, 40)
+VERSION = (0, 0, 43)
 
 __version__ = '.'.join(map(str, VERSION))

examples/add_custom_field_options.py

@@ -0,0 +1,42 @@
+
+#!/usr/bin/env python
+
+import argparse
+import json
+import logging
+import sys
+
+from blackduck.HubRestApi import HubInstance
+
+parser = argparse.ArgumentParser("Modify a custom field")
+parser.add_argument("object", choices=["BOM Component", "Component", "Component Version", "Project", "Project Version"], help="The object that the custom field should be attached to")
+parser.add_argument("field_id", help="The ID of the custom field to modify")
+parser.add_argument("-o", "--options", action='append', nargs=2, metavar=('label', 'position'), help="The options to add. To add more than one option repeat the -o option, supply a label and position for each possible selection. Used for DROPDOWN, MULTISELECT, and RADIO field types.")
+args = parser.parse_args()
+
+
+logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG)
+logging.getLogger("requests").setLevel(logging.DEBUG)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+options = [{"label": io[0], "position": io[1]} for io in args.options]
+
+hub = HubInstance()
+
+# delete all custom fields for the specified object type
+custom_fields = hub.get_custom_fields(args.object).get('items', [])
+for custom_field in custom_fields:
+    url = custom_field['_meta']['href']
+    field_id = url.split("/")[-1]
+    if field_id == args.field_id:
+        field_obj = hub.execute_get(url).json()
+
+        options_url = hub.get_link(field_obj, "custom-field-option-list")
+        for option in options:
+            response = hub.execute_post(options_url, data=option)
+            if response.status_code == 201:
+                print("Successfully added option {} to custom field {}".format(option, url))
+            else:
+                print("Failed to add option {} for custom field {}, status code: {}".format(
+                    option, url, response.status_code))
+

examples/create_fix_it_message.py

@@ -0,0 +1,103 @@
+#!/usr/bin/env python
+
+import argparse
+from datetime import datetime
+import logging
+import json
+import sys
+
+parser = argparse.ArgumentParser("Process the JSON output from get_bom_component_policy_violations.py to create a FIX IT message that guides the project team how to resolve the issues identified through policy rule violations")
+parser.add_argument("-f", "--policy_violations_file", help="By default, program reads JSON doc from stdin, but you can alternatively give a file name")
+parser.add_argument("-o", "--output_file", help="By default, the fix it message is written to stdout. Use this option to instead write to a file")
+
+args = parser.parse_args()
+
+logging.basicConfig(format='%(asctime)s%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG)
+logging.getLogger("requests").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+if args.policy_violations_file:
+    with open(args.policy_violations_file, 'r') as pvf:
+        policy_violations = json.load(pvf)
+else:
+    policy_violations = json.load(sys.stdin)
+
+project_name = policy_violations['project']['name']
+version_label = policy_violations['version']['versionName']
+version_url = policy_violations['version']['_meta']['href']
+version_license = policy_violations['version']['license']['licenseDisplay']
+if version_license.lower() == "unknown license":
+    license_statement = """This project version does not appear to have a declared license. 
+        You should probably declare one.. You can find more info on how to choose the appropriate license
+        for your project <a href=https://our-company/choosing-license>here</a>"""
+else:
+    license_statement = "This project version is governed by the {} license.".format(version_license)
+
+html = """
+<html>
+    <body>
+        <p>This is a summary of the policy violations found in project {}, version {}. It was created {}. 
+        This summary provides guidance on what to do to resolve these violations. </p>
+        <p>{}</p> 
+        <p>You can view the project version and more details regarding it
+        on Black Duck <a href={}>here</a></p>
+        <p>For more information on SCA (Software Composition Analysis) and 
+        how it fits into our SDLC go to <a href=https://our-company/sca-info>https://our-company/sca-info</a></p>
+
+        <table border=1>
+            <tr>
+                <th>Component</th>
+                <th>Version</th>
+                <th>Component License</th>
+                <th>URL</th>
+                <th>Violation</th>
+                <th>Guidance to Fix</th>
+                <th>Overrideable</th>
+                <th>Severity</th>
+            </tr>
+""".format(project_name, version_label, datetime.now(), license_statement, version_url)
+
+del policy_violations['project']
+del policy_violations['version']
+
+for violation_info in policy_violations.values():
+    component_name = violation_info['bom_component']['componentName']
+    component_version = violation_info['bom_component'].get('componentVersionName')
+    component_url = violation_info['bom_component']['component']
+    # if component-version URL is available use it, but if not revert to the component URL
+    component_version_url = violation_info['bom_component'].get('componentVersion', component_url)
+    component_license = ",".join([l['licenseDisplay'] for l in violation_info['bom_component']['licenses']])
+    policies_in_violation = violation_info['policies_in_violation']['totalCount']
+    for policy_violation in violation_info['policies_in_violation']['items']:
+        pv_name = policy_violation['name']
+        pv_description = policy_violation['description']
+        overridable = policy_violation['overridable']
+        severity = policy_violation['severity']
+        html += """
+            <tr>
+                <td>{}</td>
+                <td>{}</td>
+                <td>{}</td>
+                <td><a href={}>click here to view more info on this component from Black Duck</a></td>
+                <td>{}</td>
+                <td>{}</td>
+                <td>{}</td>
+                <td>{}</td>
+            </tr>
+        """.format(component_name, component_version, component_license, component_version_url, pv_name, pv_description, overridable, severity)
+
+html += """
+        </table>
+    </body>
+</html>
+"""
+
+if args.output_file:
+    output_file = open(args.output_file, 'w')
+else:
+    output_file = sys.stdout
+
+output_file.write(html)
+
+
+

examples/delete_custom_field_option.py

@@ -0,0 +1,42 @@
+
+#!/usr/bin/env python
+
+import argparse
+import json
+import logging
+import sys
+
+from blackduck.HubRestApi import HubInstance
+
+parser = argparse.ArgumentParser("Modify a custom field option")
+parser.add_argument("object", choices=["BOM Component", "Component", "Component Version", "Project", "Project Version"], help="The object that the custom field should be attached to")
+parser.add_argument("field_id", help="The ID of the custom field to modify")
+parser.add_argument("option_id", help="The ID of the custom field option to modify")
+args = parser.parse_args()
+
+
+logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG)
+logging.getLogger("requests").setLevel(logging.DEBUG)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
+hub = HubInstance()
+
+# delete all custom fields for the specified object type
+custom_fields = hub.get_custom_fields(args.object).get('items', [])
+for custom_field in custom_fields:
+    field_url = custom_field['_meta']['href']
+    field_id = field_url.split("/")[-1]
+    if field_id == args.field_id:
+        field_obj = hub.execute_get(field_url).json()
+
+        options_url = hub.get_link(field_obj, "custom-field-option-list")
+        options = hub.execute_get(options_url).json().get('items', [])
+        for option in options:
+            option_url = option['_meta']['href']
+            option_id = option_url.split("/")[-1]
+            if option_id == args.option_id:
+                response = hub.execute_delete(option_url)
+                if response.status_code == 204:
+                    print("Deleted option {}".format(option_url))
+                else:
+                    print("Failed to delete option {}, status code: {}".format(option_url, response.status_code))

examples/generate_notices_report_for_project_version.py

@@ -10,42 +10,51 @@
 
 import argparse
 import json
+import logging
+import sys
 import time
 import zipfile
 
 parser = argparse.ArgumentParser("A program to generate the notices file for a given project-version")
 parser.add_argument("project_name")
 parser.add_argument("version_name")
-parser.add_argument("--zip_file_name", default="notices_report.zip")
-parser.add_argument("--reports",
-	default="version,scans,components,vulnerabilities,source", 
-	help="Comma separated list (no spaces) of the reports to generate - version, scans, components, vulnerabilities, source, and cryptography reports (default: all, except cryptography")
-parser.add_argument('--format', default='TEXT', choices=["HTML", "TEXT"], help="Report format - choices are TEXT or HTML")
+parser.add_argument('-f', "--file_name_base", default="notices_report", help="Base file name to write the report data into. If the report format is TEXT a .zip file will be created, otherwise a .json file")
+parser.add_argument('-r', '--report_format', default='TEXT', choices=["JSON", "TEXT"], help="Report format - choices are TEXT or HTML")
 
 args = parser.parse_args()
 
 hub = HubInstance()
 
-# TODO: Promote this to the API?
+logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG)
+
 class FailedReportDownload(Exception):
 	pass
 
-def download_report(location, filename, retries=4):
+def download_report(location, file_name_base, retries=10):
 	report_id = location.split("/")[-1]
 
 	if retries:
-		print("Retrieving generated report from {}".format(location))
-		response = hub.download_report(report_id)
+		logging.debug("Retrieving generated report from {}".format(location))
+		# response = hub.download_report(report_id)
+		response, report_format = hub.download_notification_report(location)
 		if response.status_code == 200:
-			with open(filename, "wb") as f:
-				f.write(response.content)
-			print("Successfully downloaded zip file to {} for report {}".format(filename, report_id))
+			if report_format == "TEXT":
+				filename = file_name_base + ".zip"
+				with open(filename, "wb") as f:
+					f.write(response.content)
+			else:
+				# JSON format
+				filename = file_name_base + ".json"
+				with open(filename, "w") as f:
+					json.dump(response.json(), f, indent=3)
+			logging.info("Successfully downloaded json file to {} for report {}".format(
+					filename, report_id))
 		else:
-			print("Failed to retrieve report {}".format(report_id))
-			print("Probably not ready yet, waiting 5 seconds then retrying...")
+			logging.warning("Failed to retrieve report {}".format(report_id))
+			logging.warning("Probably not ready yet, waiting 5 seconds then retrying (remaining retries={}".format(retries))
 			time.sleep(5)
 			retries -= 1
-			download_report(location, filename, retries)
+			download_report(location, file_name_base, retries)
 	else:
 		raise FailedReportDownload("Failed to retrieve report {} after multiple retries".format(report_id))
 
@@ -54,22 +63,23 @@ def download_report(location, filename, retries=4):
 if project:
 	version = hub.get_version_by_name(project, args.version_name)
 
-	response = hub.create_version_notices_report(version, args.format)
+	response = hub.create_version_notices_report(version, args.report_format)
 
 	if response.status_code == 201:
-		print("Successfully created reports ({}) for project {} and version {}".format(
-			args.reports, args.project_name, args.version_name))
+		logging.info("Successfully created notices report in {} format for project {} and version {}".format(
+			args.report_format, args.project_name, args.version_name))
 		location = response.headers['Location']
-		download_report(location, args.zip_file_name)
+		download_report(location, args.file_name_base)
+
 
 		# Showing how you can interact with the downloaded zip and where to find the
 		# output content. Uncomment the lines below to see how it works.
 
-		# with zipfile.ZipFile(zip_file_name, 'r') as zipf:
+		# with zipfile.ZipFile(zip_file_name_base, 'r') as zipf:
 		# 	with zipf.open("{}/{}/version-license.txt".format(args.project_name, args.version_name), "r") as license_file:
 		# 		print(license_file.read())
 	else:
-		print("Failed to create reports for project {} version {}, status code returned {}".format(
+		logging.error("Failed to create reports for project {} version {}, status code returned {}".format(
 			args.project_name, args.version_name, response.status_code))
 else:
-	print("Did not find project with name {}".format(args.project_name))
+	logging.warning("Did not find project with name {}".format(args.project_name))