diff --git a/.gitignore b/.gitignore index 894a44cc..1c20e402 100644 --- a/.gitignore +++ b/.gitignore @@ -86,6 +86,7 @@ celerybeat-schedule .venv env/ venv/ +virtualenv/ ENV/ env.bak/ venv.bak/ diff --git a/README.md b/README.md index abcb1d05..5f62fb1f 100644 --- a/README.md +++ b/README.md @@ -13,18 +13,18 @@ pip install blackduck ``` ```python -from blackduck.HubRestApi import HubInstance +from blackduck import Client import json -username = "sysadmin" -password = "your-password" -urlbase = "https://ec2-34-201-23-208.compute-1.amazonaws.com" +bd = Client( + token=os.environ.get('blackduck_token', 'YOUR TOKEN HERE'), + base_url='https://your.blackduck.url' #!important! no trailing slash + #, verify=False # if required +) -hub = HubInstance(urlbase, username, password, insecure=True) +for project in bd.get_projects(): + print(project.get('name') -projects = hub.get_projects() - -print(json.dumps(projects.get('items', []))) ``` ### Examples diff --git a/blackduck/Authentication.py b/blackduck/Authentication.py index 26d38097..0b74ab77 100644 --- a/blackduck/Authentication.py +++ b/blackduck/Authentication.py @@ -1,7 +1,95 @@ -import logging +''' + +Created on Dec 23, 2020 +@author: ar-calder + +''' + import requests +import logging import json -from operator import itemgetter -import urllib.parse +from datetime import datetime, timedelta + +logger = logging.getLogger(__name__) + +class BearerAuth(requests.auth.AuthBase): + + from .Exceptions import http_exception_handler + + def __init__( + self, + session=None, + token=None, + base_url=None, + verify=True, + timeout=15, + ): + + if any(arg == False for arg in (token, base_url)): + raise ValueError( + 'token & base_url are required' + ) + + self.verify=verify + self.client_token = token + self.auth_token = None + self.csrf_token = None + self.valid_until = datetime.utcnow() + + self.auth_url = requests.compat.urljoin(base_url, '/api/tokens/authenticate') + self.session = session or requests.session() + self.timeout = timeout + + + def __call__(self, request): + if not self.auth_token or self.valid_until < datetime.utcnow(): + # If authentication token not set or no longer valid + self.authenticate() + + request.headers.update({ + "authorization" : f"bearer {self.auth_token}", + "X-CSRF-TOKEN" : self.csrf_token + }) + + return request + + + def authenticate(self): + if not self.verify: + requests.packages.urllib3.disable_warnings() + # Announce this on every auth attempt, as a little incentive to properly configure certs + logger.warn("ssl verification disabled, connection insecure. do NOT use verify=False in production!") + + try: + response = self.session.request( + method='POST', + url=self.auth_url, + headers = { + "Authorization" : f"token {self.client_token}" + }, + verify=self.verify, + timeout=self.timeout + ) + + if response.status_code / 100 != 2: + self.http_exception_handler( + response=response, + name="authenticate" + ) + + content = response.json() + self.csrf_token = response.headers.get('X-CSRF-TOKEN') + self.auth_token = content.get('bearerToken') + self.valid_until = datetime.utcnow() + timedelta(milliseconds=int(content.get('expiresInMilliseconds', 0))) -logger = logging.getLogger(__name__) \ No newline at end of file + # Do not handle exceptions - just just more details as to possible causes + # Thus we do not catch a JsonDecodeError here even though it may occur + # - no futher details to give. + except requests.exceptions.ConnectTimeout as connect_timeout: + logger.critical(f"could not establish a connection within {self.timeout}s, this may be indicative of proxy misconfiguration") + raise connect_timeout + except requests.exceptions.ReadTimeout as read_timeout: + logger.critical(f"slow or unstable connection, consider increasing timeout (currently set to {self.timeout}s)") + raise read_timeout + else: + logger.info(f"success: auth granted until {self.valid_until} UTC") diff --git a/blackduck/Client.py b/blackduck/Client.py new file mode 100644 index 00000000..1ab20e02 --- /dev/null +++ b/blackduck/Client.py @@ -0,0 +1,68 @@ +''' +Created on Dec 23, 2020 +@author: ar-calder + +Wrapper for common HUB API queries. +Upon initialization Bearer token is obtained and used for all subsequent calls. +Token will auto-renew on timeout. +''' + +from .Utils import find_field, safe_get +from .Authentication import BearerAuth +import logging +import requests +logger = logging.getLogger(__name__) + +class Client: + VERSION_DISTRIBUTION=["EXTERNAL", "SAAS", "INTERNAL", "OPENSOURCE"] + VERSION_PHASES = ["PLANNING", "DEVELOPMENT", "PRERELEASE", "RELEASED", "DEPRECATED", "ARCHIVED"] + PROJECT_VERSION_SETTINGS = ['nickname', 'releaseComments', 'versionName', 'phase', 'distribution', 'releasedOn'] + + from .Exceptions import( + http_exception_handler + ) + + from .ClientCore import ( + _request, _get_items, _get_resource_href, get_resource, list_resources, _get_base_resource_url, get_base_resource, _get_parameter_string + ) + + def __init__( + self, + *args, + token=None, + base_url=None, + session=None, + auth=None, + verify=True, + timeout=15, + **kwargs): + + self.verify=verify + self.timeout=int(timeout) + self.base_url=base_url + self.session = session or requests.session() + self.auth = auth or BearerAuth( + session = self.session, + token=token, + base_url=base_url, + verify=self.verify + ) + + def print_methods(self): + import inspect + for fn in inspect.getmembers(self, predicate=inspect.ismember): + print(fn[0]) + + # Example for projects + def get_projects(self, parameters=[], **kwargs): + return self._get_items( + method='GET', + # url unlikely to change hence is_public=false (faster). + url= self._get_base_resource_url('projects', is_public=False), + name="project", + **kwargs + ) + + def get_project_by_name(self, project_name, **kwargs): + projects = self.get_projects(**kwargs) + return find_field(projects, 'name', project_name) diff --git a/blackduck/ClientCore.py b/blackduck/ClientCore.py new file mode 100644 index 00000000..b3e2f942 --- /dev/null +++ b/blackduck/ClientCore.py @@ -0,0 +1,176 @@ +''' +Created on Dec 23, 2020 +@author: ar-calder + +''' + +import logging +import requests +import json + +from .Utils import find_field, safe_get +logger = logging.getLogger(__name__) + +def _request( + self, + method, + url, + name='', + parameters=[], + **kwargs + ): + """[summary] + + Args: + method ([type]): [description] + url ([type]): [description] + name (str, optional): name of the reqested resource. Defaults to ''. + + Raises: + connect_timeout: often indicative of proxy misconfig + read_timeout: often indicative of slow connection + + Returns: + json/dict/list: requested object, json decoded. + """ + + headers = { + 'accept' : 'application/json' + } + headers.update(kwargs.pop('headers', dict())) + + if parameters: + url += self._get_parameter_string(parameters) + + try: + response = self.session.request( + method=method, + url=url, + headers=headers, + verify=self.verify, + auth=self.auth, + **kwargs + ) + + if response.status_code / 100 != 2: + self.http_exception_handler( + response=response, + name=name + ) + + response_json = response.json() + + # Do not handle exceptions - just just more details as to possible causes + # Thus we do not catch a JsonDecodeError here even though it may occur + except requests.exceptions.ConnectTimeout as connect_timeout: + logger.critical(f"could not establish a connection within {self.timeout}s, this may be indicative of proxy misconfiguration") + raise connect_timeout + except requests.exceptions.ReadTimeout as read_timeout: + logger.critical(f"slow or unstable connection, consider increasing timeout (currently set to {self.timeout}s)") + raise read_timeout + else: + return response_json + +def _get_items(self, url, method='GET', page_size=100, name='', **kwargs): + """Utility method to get 'pages' of items + + Args: + url (str): [description] + method (str, optional): [description]. Defaults to 'GET'. + page_size (int, optional): [description]. Defaults to 100. + name (str, optional): [description]. Defaults to ''. + + Yields: + [type]: [description] + """ + offset = 0 + params = kwargs.pop('params', dict()) + while True: + params.update({'offset':f"{offset}", 'limit':f"{page_size}"}) + items = self._request( + method=method, + url=url, + params=params, + name=name, + **kwargs + ).get('items', list()) + + for item in items: + yield item + + if len(items) < page_size: + # This will be true if there are no more 'pages' to view + break + + offset += page_size + + +def _get_resource_href(self, resources, resource_name): + """Utility function to get url for a given resource_name + + Args: + resources (dict/json): [description] + resource_name (str): [description] + + Raises: + KeyError: on key not found + + Returns: + str: url to named resource + """ + res = find_field( + data_to_filter=safe_get(resources, '_meta', 'links'), + field_name='rel', + field_value=resource_name + ) + + if None == res: + raise KeyError(f"'{self.get_resource_name(resources)}' object has no such key '{resource_name}'") + return safe_get(res, 'href') + +def get_resource(self, bd_object, resource_name, iterable=True, is_public=True, **kwargs): + """Generic function to facilitate subresource fetching + + Args: + bd_object (dict/json): [description] + resource_name (str): [description] + iterable (bool, optional): [description]. Defaults to True. + is_public (bool, optional): [description]. Defaults to True. + + Returns: + dict/json: named resource object + """ + url = self._get_resource_href(resources=bd_object, resource_name=resource_name) if is_public else self.get_url(bd_object) + f"/{resource_name}" + fn = self._get_items if iterable else self._request + return fn( + method='GET', + url=url, + name=resource_name, + **kwargs + ) + +def list_resources(self, bd_object): + return [res.get('rel') for res in safe_get(bd_object, '_meta', 'links')] + +def _get_base_resource_url(self, resource_name, is_public=True, **kwargs): + if is_public: + resources = self._request( + method="GET", + url=self.base_url + f"/api/", + name='_get_base_resource_url', + **kwargs + ) + return resources.get(resource_name, "") + else: + return self.base_url + f"/api/{resource_name}" + +def get_base_resource(self, resource_name, is_public=True, **kwargs): + return self._request( + method='GET', + url=self._get_base_resource_url(resource_name, is_public=is_public, **kwargs), + name='get_base_resource', + **kwargs + ) + +def _get_parameter_string(self, parameters=list()): + return '?' + '&'.join(parameters) if parameters else '' diff --git a/blackduck/Exceptions.py b/blackduck/Exceptions.py index 61f71fcd..fa8c61cd 100644 --- a/blackduck/Exceptions.py +++ b/blackduck/Exceptions.py @@ -32,7 +32,7 @@ class EndpointNotFound(Exception): class UnacceptableContentType(Exception): pass -def exception_handler(self, response, name): +def http_exception_handler(self, response, name): error_codes = { 404 : EndpointNotFound, 406 : UnacceptableContentType diff --git a/blackduck/Utils.py b/blackduck/Utils.py index f4e010b2..1253d158 100644 --- a/blackduck/Utils.py +++ b/blackduck/Utils.py @@ -13,7 +13,7 @@ logger = logging.getLogger(__name__) -def iso8061_to_date(iso_string, with_zone=False): +def iso8601_to_date(iso_string, with_zone=False): """Utility function to convert iso_8061 formatted string to datetime object, optionally accounting for timezone Args: @@ -31,6 +31,12 @@ def iso8061_to_date(iso_string, with_zone=False): date = date + datetime.timedelta(minutes=minutes) return date +def iso8601_timespan(days_ago, from_date=datetime.utcnow(), delta=timedelta(weeks=1)): + curr_date = from_date - timedelta(days=days_ago) + while curr_date < from_date: + yield curr_date.isoformat('T', 'seconds') + curr_date += delta + def min_iso8061(): """Utility wrapper for iso8061_to_date which provides minimum date (for comparison purposes). @@ -50,7 +56,7 @@ def find_field(data_to_filter, field_name, field_value): Returns: object: object if found or None. """ - return next(filter(lambda d: d.get(field) == field_value, data_to_filter), None) + return next(filter(lambda d: d.get(field_name) == field_value, data_to_filter), None) def safe_get(obj, *keys): """Utility function to safely perform multiple get's on a dict. diff --git a/blackduck/__init__.py b/blackduck/__init__.py index 5fbbeeb7..5d52db1e 100644 --- a/blackduck/__init__.py +++ b/blackduck/__init__.py @@ -1,2 +1,3 @@ from .HubRestApi import HubInstance +from .Client import Client diff --git a/test/demo_client.py b/test/demo_client.py new file mode 100644 index 00000000..3f9f42a2 --- /dev/null +++ b/test/demo_client.py @@ -0,0 +1,74 @@ +import os +import requests +from requests.adapters import HTTPAdapter +import logging + +logging.basicConfig( + level=logging.INFO, + format='[%(asctime)s] {%(module)s:%(lineno)d} %(levelname)s - %(message)s' +) + +# create http adapter with exponential backoff (for unstable and/or slow connections) +http_adapter = HTTPAdapter( + max_retries=requests.packages.urllib3.util.retry.Retry( + total=5, + backoff_factor=10, + status_forcelist=[429,500,502,503,504] + ) +) +custom_session = requests.session() +custom_session.mount('http://', http_adapter) +custom_session.mount('https://', http_adapter) + +# use os env proxy settings, if any +custom_session.proxies.update({ + 'http' : os.environ.get('http_proxy',''), + 'https' : os.environ.get('http_proxy', '') +}) + + +# Brief demo +from datetime import datetime, timedelta +import blackduck + +def vulns_in_all_project_versions_components(bd): + for project in bd.get_projects(): + for version in bd.get_resource(project, 'versions'): + for component in bd.get_resource(version, 'components'): + for vulnerability in bd.get_resource(component, 'vulnerabilities'): + print(f"{project.get('name')}-{version.get('versionName')} [{component.get('componentName')}] has {vulnerability.get('severity')} severity vulnerability '{vulnerability.get('name')}'") + +def list_project_subresources(bd): + for project in bd.get_projects(): + subresources = bd.list_resources(project) + print(f"projects has the following subresources: {', '.join(subresources)}") + return + + +def projects_added_at_4_week_intervals(bd): + last_count = 0 + count = 0 + print("Projects added, in 4 week intervals:") + for timestamp in blackduck.Utils.iso8601_timespan(days_ago=365, delta=timedelta(weeks=4)): + last_count=count + count=0 + for project in bd.get_projects(): + created_at = blackduck.Utils.iso8601_to_date(project.get('createdAt')) + count += (created_at <= blackduck.Utils.iso8601_to_date(timestamp)) + + print(f"{count-last_count} projects as of {timestamp}") + +bd = blackduck.Client( + token=os.environ.get('blackduck_token', 'YOUR TOKEN HERE'), + base_url='https://your.blackduck.url', #!important! no trailing slash + session=custom_session + # verify=False # if required +) + +# If disabling warnings, don't do so at the library level: +requests.packages.urllib3.disable_warnings() + +# Various examples: +# vulns_in_all_project_versions_components(bd) +projects_added_at_4_week_intervals(bd) +# list_project_subresources(bd) \ No newline at end of file