Merge pull request #68 from blackducksoftware/gsnyder/various-fixes

refactoring to use public APIs whenever possible

Author: Glenn Snyder

blackduck/HubRestApi.py

@@ -472,10 +472,32 @@ def get_vulnerabilities(self, vulnerability, parameters={}):
         return response.json()
 
     def get_vulnerability_affected_projects(self, vulnerability):
-        url = self.config['baseurl'] + "/api/v1/composite/vulnerability"+ "/{}".format(vulnerability) 
+        url = self._get_vulnerabilities_url() + "/{}/affected-projects".format(vulnerability) 
+        custom_headers = {'Accept': 'application/vnd.blackducksoftware.vulnerability-4+json'}
+        response = self.execute_get(url, custom_headers=custom_headers)
+        return response.json()
+
+    # TODO: Refactor this, i.e. use get_link method?
+    def get_vulnerable_bom_components(self, version_obj, limit=9999):
+        url = "{}/vulnerable-bom-components".format(version_obj['_meta']['href'])
+        custom_headers = {'Accept': 'application/vnd.blackducksoftware.bill-of-materials-6+json'}
+        param_string = self._get_parameter_string({'limit': limit})
+        url = "{}{}".format(url, param_string)
+        response = self.execute_get(url, custom_headers=custom_headers)
+        return response.json()
+
+    # TODO: Remove or refactor this
+    def get_component_remediation(self, bom_component):
+        url = "{}/remediating".format(bom_component['componentVersion'])
+        logging.debug("Url for getting remediation info is : {}".format(url))
         response = self.execute_get(url)
         return response.json()
 
+    ##
+    #
+    # Lookup Black Duck (Hub) KB info given Protex KB info
+    #
+    ##
     def find_component_info_for_protex_component(self, protex_component_id, protex_component_release_id):
         '''Will return the Hub component corresponding to the protex_component_id, and if a release (version) id
         is given, the response will also include the component-version. Returns an empty list if there were
@@ -495,24 +517,6 @@ def find_component_info_for_protex_component(self, protex_component_id, protex_c
         component_list_d = response.json()
         return response.json()
 
-    def get_vulnerable_bom_components(self, version_obj, limit=9999):
-        url = "{}/vulnerable-bom-components".format(version_obj['_meta']['href'])
-        custom_headers = {'Content-Type': 'application/vnd.blackducksoftware.bill-of-materials-4+json'}
-        param_string = self._get_parameter_string({'limit': limit})
-        url = "{}{}".format(url, param_string)
-        response = self.execute_get(url, custom_headers=custom_headers)
-        if response.status_code == 200:
-            vulnerable_bom_components = response.json()
-            return vulnerable_bom_components
-        else:
-            logging.warning("Failed to retrieve vulnerable bom components for project {}, status code {}".format(
-                version_obj, response.status_code))
-
-    def get_component_remediation(self, bom_component):
-        url = "{}/remediating".format(bom_component['componentVersion'])
-        logging.debug("Url for getting remediation info is : {}".format(url))
-        response = self.execute_get(url)
-        return response.json()
 
     ##
     #
@@ -1146,6 +1150,25 @@ def get_project_roles(self):
         all_project_roles = self.get_roles(parameters={"filter":"scope:project"})
         return all_project_roles['items']
 
+    def get_version_scan_info(self, version_obj):
+        url = self.get_link(version_obj, "codelocations")
+        custom_headers = {'Accept': 'application/vnd.blackducksoftware.project-detail-5+json'}
+        response = self.execute_get(url, custom_headers=custom_headers)
+        code_locations = response.json().get('items', [])
+        if code_locations:
+            scan_info = {
+                'most_recent_scan': max([cl['updatedAt'] for cl in code_locations]),
+                'oldest_scan': min([cl['createdAt'] for cl in code_locations]),
+                'number_scans': len(code_locations)
+            }
+        else:
+            scan_info = {
+                'most_recent_scan': None,
+                'oldest_scan': None,
+                'number_scans': None
+            }
+        return scan_info
+
     ###
     #
     # Add project version as a component to another project
@@ -1237,18 +1260,7 @@ def get_codelocations(self, limit=100, unmapped=False, parameters={}):
         url = self.get_apibase() + "/codelocations" + paramstring
         headers['Accept'] = 'application/vnd.blackducksoftware.scan-4+json'
         response = requests.get(url, headers=headers, verify = not self.config['insecure'])
-        if response.status_code == 200:
-            jsondata = response.json()
-            if unmapped:
-                jsondata['items'] = [s for s in jsondata['items'] if 'mappedProjectVersion' not in s]
-                jsondata['totalCount'] = len(jsondata['items'])
-            return jsondata
-        elif response.status_code == 403:
-            logging.warning("Failed to retrieve code locations (aka scans) probably due to lack of permissions, status code {}".format(
-                response.status_code))
-        else:
-            logging.error("Failed to retrieve code locations (aka scans), status code {}".format(
-                response.status_code))
+        return response.json()
 
     def get_codelocation_scan_summaries(self, code_location_id = None, code_location_obj = None, limit=100):
         '''Retrieve the scans (aka scan summaries) for the given location. You can give either
@@ -1269,7 +1281,7 @@ def get_codelocation_scan_summaries(self, code_location_id = None, code_location
         return jsondata
 
     def delete_unmapped_codelocations(self, limit=1000):
-        code_locations = self.get_codelocations(limit, True).get('items', [])
+        code_locations = self.get_codelocations(limit=limit, unmapped=True).get('items', [])
 
         for c in code_locations:
             scan_summaries = self.get_codelocation_scan_summaries(code_location_obj = c).get('items', [])
@@ -1509,6 +1521,18 @@ def get_health_checks(self):
         url = self.get_urlbase() + "/api/health-checks/liveness"
         return self.execute_get(url)
 
+    ##
+    #
+    # Jobs
+    #
+    ##
+    def get_jobs(self, parameters={}):
+        url = self.get_apibase() + "/jobs"
+        url = url + self._get_parameter_string(parameters)
+        custom_headers = {'Accept': 'application/vnd.blackducksoftware.status-4+json'}
+        response = self.execute_get(url, custom_headers=custom_headers)
+        return response.json()
+
     ##
     #
     # Job Statistics

blackduck/__version__.py

@@ -1,3 +1,3 @@
-VERSION = (0, 0, 36)
+VERSION = (0, 0, 37)
 
 __version__ = '.'.join(map(str, VERSION))

examples/check_scan_jobs_status.py

@@ -34,19 +34,7 @@ def initialize(cls):
 
         if not cls.jobs:
             logging.debug("retrieving jobs")
-            cls.jobs = cls._get_all_jobs()
-
-    @classmethod
-    def _get_all_jobs(cls):
-        # See https://jira.dc1.lan/browse/HUB-14263 - Rest API for Job status
-        # The below is using a private endpoint that could change on any release and break this code
-        jobs_url = cls.hub.get_urlbase() + "/api/v1/jobs?limit={}".format(10000)
-        response = cls.hub.execute_get(jobs_url)
-        jobs = []
-        if response.status_code == 200:
-            return response.json().get('items', [])
-        else:
-            raise Exception("Failed to retrieve jobs, status code: {}".format(response.status_code))
+            cls.jobs = cls.hub.get_jobs().get('items', [])
 
     def __init__(self, code_location_obj):
         self.initialize()

examples/get_scans_or_codelocations.py

@@ -9,16 +9,24 @@
 
 import argparse
 import json
+import logging
 from pprint import pprint
+import sys
 
-from blackduck.HubRestApi import HubInstance
+from blackduck.HubRestApi import HubInstance, object_id
 
 parser = argparse.ArgumentParser("Retrieve scans (aka code locations) from the Black Duck system")
-parser.add_argument("--name", type=str, default=None, help="Filter by name")
+parser.add_argument("-n", "--name", type=str, default=None, help="Filter by name")
 parser.add_argument("--unmapped", action='store_true', help="Set this to see any scans (aka code locations) that are not mapped to any project-version")
+parser.add_argument("-s", "--scan_summaries", action='store_true', help="Set this option to include scan summaries")
+parser.add_argument("-d", "--scan_details", action='store_true')
 
 args = parser.parse_args()
 
+logging.basicConfig(format='%(asctime)s:%(levelname)s:%(message)s', stream=sys.stderr, level=logging.DEBUG)
+logging.getLogger("requests").setLevel(logging.WARNING)
+logging.getLogger("urllib3").setLevel(logging.WARNING)
+
 hub = HubInstance()
 
 if args.name:
@@ -27,8 +35,24 @@
 	parameters={}
 
 if args.unmapped:
-	scans = hub.get_codelocations(limit=10000, unmapped=True, parameters=parameters)
+	code_locations = hub.get_codelocations(limit=10000, unmapped=True, parameters=parameters)
 else:
-	scans = hub.get_codelocations(limit=10000, parameters=parameters)
-
-print(json.dumps(scans))
+	code_locations = hub.get_codelocations(limit=10000, parameters=parameters)
+
+code_locations = code_locations.get('items', [])
+
+if args.scan_summaries:
+    for code_location in code_locations:
+        scan_summaries = hub.get_codelocation_scan_summaries(code_location_obj=code_location).get('items', [])
+        code_location['scan_summaries'] = scan_summaries
+        if args.scan_details:
+            for scan in scan_summaries:
+                scan_id = object_id(scan)
+                # This uses a private API endpoint that can, and probably will, break in the future
+                # HUB-15330 is the (internal) JIRA ticket # asking that the information in this endpoint
+                # be made part of the public API
+                url = hub.get_apibase() + "/v1/scans/{}".format(scan_id)
+                scan_details = hub.execute_get(url).json()
+                scan['scan_details'] = scan_details
+
+print(json.dumps(code_locations))

examples/print_vulnerability_affected_projects.py

@@ -19,24 +19,29 @@
 parser.add_argument("vulnerability", help="A CVE or BDSA number, e.g. CVE-2016-4009")
 args = parser.parse_args()
 
-affected_projects = hub.get_vulnerability_affected_projects(args.vulnerability.upper())
+affected_projects = hub.get_vulnerability_affected_projects(args.vulnerability.upper()).get('items', [])
 
-if 'totalCount' in affected_projects and affected_projects['totalCount'] > 0:
+if affected_projects:
 	ttable = [[
 		"project-name", 
 		"version", 
 		"phase", 
 		"distribution", 
-		"last-bom-update", 
+		"scan-info", 
 		"Owner Name", 
 		"Owner email"]]
 
-	for affected_project in affected_projects['items']:
+	for affected_project in affected_projects:
 		# Get the Owner info for the project
-		project_json = hub.get_project_by_id(affected_project['project']['id'])
-		if 'projectOwner' in project_json:
-			owner_response = hub.execute_get(project_json['projectOwner'])
+		# project_json = hub.get_project_by_id(affected_project['project']['id'])
+		project_url = affected_project['project']
+		custom_headers = {'Accept': 'application/vnd.blackducksoftware.project-detail-4+json'}
+		project_details = hub.execute_get(project_url, custom_headers=custom_headers).json()
+		if 'projectOwner' in project_details:
+			custom_headers = {'Accept': 'application/vnd.blackducksoftware.user-4+json'}
+			owner_response = hub.execute_get(project_details['projectOwner'], custom_headers=custom_headers)
 			owner_json = owner_response.json()
+
 			if 'firstName' in owner_json and 'lastName' in owner_json:
 				owner_name = owner_json['firstName'] + ' ' + owner_json['lastName']
 			else:
@@ -48,26 +53,28 @@
 		else:
 			owner_name = owner_email = "None supplied"
 
-		project_name = affected_project['project']['name']
-		version = affected_project['release']['version']
+		project_name = affected_project['projectName']
+		version = affected_project['projectVersionName']
 
 		# Development phase does not appear to be in the payload returned by the affected projects
 		# endpoint so we need to fetch it from the project-version endpoint
-		project_id = affected_project['project']['id']
-		version_id = affected_project['release']['id']
-
-		project_version_info = hub.get_version_by_id(project_id, version_id)
+		version_url = affected_project['projectVersion']
+		custom_headers = {'Accept': 'application/vnd.blackducksoftware.project-detail-5+json'}
+		project_version_info = hub.execute_get(version_url, custom_headers=custom_headers).json()
 
 		phase = project_version_info['phase']
 		distribution = project_version_info['distribution']
-		last_bom_update = project_version_info['lastBomUpdateDate']
+		scan_info = hub.get_version_scan_info(project_version_info)
+		separator = "\n"
+		scan_info_str = separator.join("{}: {}".format(k,v) for k,v in scan_info.items())
+		# last_bom_update = project_version_info['lastBomUpdateDate']
 
 		ttable.append([
 			project_name, 
 			version, 
 			phase, 
 			distribution, 
-			last_bom_update, 
+			scan_info_str, 
 			owner_name, 
 			owner_email])
 	print(AsciiTable(ttable).table)