diff --git a/analyzers/Qintel/qintel_helper.py b/analyzers/Qintel/qintel_helper.py new file mode 100644 index 000000000..47106f78a --- /dev/null +++ b/analyzers/Qintel/qintel_helper.py @@ -0,0 +1,263 @@ +# Copyright (c) 2009-2021 Qintel, LLC +# Licensed under Apache 2.0 (https://www.apache.org/licenses/LICENSE-2.0.txt) + +from urllib.request import Request, urlopen +from urllib.parse import urlencode +from urllib.error import HTTPError +from time import sleep +from json import loads +import os +from copy import deepcopy +from datetime import datetime, timedelta +from gzip import GzipFile + +VERSION = '1.0.1' +USER_AGENT = 'integrations-helper' +MAX_RETRY_ATTEMPTS = 5 + +DEFAULT_HEADERS = { + 'User-Agent': f'{USER_AGENT}/{VERSION}' +} + +REMOTE_MAP = { + 'pmi': 'https://api.pmi.qintel.com', + 'qwatch': 'https://api.qwatch.qintel.com', + 'qauth': 'https://api.qauth.qintel.com', + 'qsentry_feed': 'https://qsentry.qintel.com', + 'qsentry': 'https://api.qsentry.qintel.com' +} + +ENDPOINT_MAP = { + 'pmi': { + 'ping': '/users/me', + 'cve': 'cves' + }, + 'qsentry_feed': { + 'anon': '/files/anonymization', + 'mal_hosting': '/files/malicious_hosting' + }, + 'qsentry': {}, + 'qwatch': { + 'ping': '/users/me', + 'exposures': 'exposures' + }, + 'qauth': {} +} + + +def _get_request_wait_time(attempts): + """ Use Fibonacci numbers for determining the time to wait when rate limits + have been encountered. + """ + + n = attempts + 3 + a, b = 1, 0 + for _ in range(n): + a, b = a + b, a + + return a + + +def _search(**kwargs): + remote = kwargs.get('remote') + max_retries = int(kwargs.get('max_retries', MAX_RETRY_ATTEMPTS)) + params = kwargs.get('params', {}) + headers = _set_headers(**kwargs) + + logger = kwargs.get('logger') + + params = urlencode(params) + url = remote + "?" + params + req = Request(url, headers=headers) + + request_attempts = 1 + while request_attempts < max_retries: + try: + return urlopen(req) + + except HTTPError as e: + response = e + + except Exception as e: + raise Exception('API connection error') from e + + if response.code not in [429, 504]: + raise Exception(f'API connection error: {response}') + + if request_attempts < max_retries: + wait_time = _get_request_wait_time(request_attempts) + + if response.code == 429: + msg = 'rate limit reached on attempt {request_attempts}, ' \ + 'waiting {wait_time} seconds' + + if logger: + logger(msg) + + else: + msg = f'connection timed out, retrying in {wait_time} seconds' + if logger: + logger(msg) + + sleep(wait_time) + + else: + raise Exception('Max API retries exceeded') + + request_attempts += 1 + + +def _set_headers(**kwargs): + headers = deepcopy(DEFAULT_HEADERS) + + if kwargs.get('user_agent'): + headers['User-Agent'] = \ + f"{kwargs['user_agent']}/{USER_AGENT}/{VERSION}" + + # TODO: deprecate + if kwargs.get('client_id') or kwargs.get('client_secret'): + try: + headers['Cf-Access-Client-Id'] = kwargs['client_id'] + headers['Cf-Access-Client-Secret'] = kwargs['client_secret'] + except KeyError: + raise Exception('missing client_id or client_secret') + + if kwargs.get('token'): + headers['x-api-key'] = kwargs['token'] + + return headers + + +def _set_remote(product, query_type, **kwargs): + remote = kwargs.get('remote') + endpoint = kwargs.get('endpoint', ENDPOINT_MAP[product].get(query_type)) + + if not remote: + remote = REMOTE_MAP[product] + + if not endpoint: + raise Exception('invalid search type') + + remote = remote.rstrip('/') + endpoint = endpoint.lstrip('/') + + return f'{remote}/{endpoint}' + + +def _process_qsentry(resp): + if resp.getheader('Content-Encoding', '') == 'gzip': + with GzipFile(fileobj=resp) as file: + for line in file.readlines(): + yield loads(line) + + +def search_pmi(search_term, query_type, **kwargs): + """ + Search PMI + + :param str search_term: Search term + :param str query_type: Query type [cve|ping] + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + kwargs['remote'] = _set_remote('pmi', query_type, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('PMI_TOKEN')) + + params = kwargs.get('params', {}) + params.update({'identifier': search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def search_qwatch(search_term, search_type, query_type, **kwargs): + """ + Search QWatch for exposed credentials + + :param str search_term: Search term + :param str search_type: Search term type [domain|email] + :param str query_type: Query type [exposures] + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + kwargs['remote'] = _set_remote('qwatch', query_type, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QWATCH_TOKEN')) + + params = kwargs.get('params', {}) + if search_type: + params.update({search_type: search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def search_qauth(search_term, **kwargs): + """ + Search QAuth + + :param str search_term: Search term + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + if not kwargs.get('endpoint'): + kwargs['endpoint'] = '/' + + kwargs['remote'] = _set_remote('qauth', None, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QAUTH_TOKEN')) + + params = kwargs.get('params', {}) + params.update({'q': search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def search_qsentry(search_term, **kwargs): + """ + Search QSentry + + :param str search_term: Search term + :param dict kwargs: extra client args [remote|token|params] + :return: API JSON response object + :rtype: dict + """ + + if not kwargs.get('endpoint'): + kwargs['endpoint'] = '/' + + kwargs['remote'] = _set_remote('qsentry', None, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN')) + + params = kwargs.get('params', {}) + params.update({'q': search_term}) + kwargs['params'] = params + + return loads(_search(**kwargs).read()) + + +def qsentry_feed(query_type='anon', feed_date=datetime.today(), **kwargs): + """ + Fetch the most recent QSentry Feed + + :param str query_type: Feed type [anon|mal_hosting] + :param dict kwargs: extra client args [remote|token|params] + :param datetime feed_date: feed date to fetch + :return: API JSON response object + :rtype: Iterator[dict] + """ + + remote = _set_remote('qsentry_feed', query_type, **kwargs) + kwargs['token'] = kwargs.get('token', os.getenv('QSENTRY_TOKEN')) + + feed_date = (feed_date - timedelta(days=1)).strftime('%Y%m%d') + kwargs['remote'] = f'{remote}/{feed_date}' + + resp = _search(**kwargs) + for r in _process_qsentry(resp): + yield r diff --git a/analyzers/Qintel/qintel_qwatch.json b/analyzers/Qintel/qintel_qwatch.json new file mode 100644 index 000000000..33d38577f --- /dev/null +++ b/analyzers/Qintel/qintel_qwatch.json @@ -0,0 +1,34 @@ +{ + "name": "Qintel_QWatch", + "version": "1.0", + "url": "None", + "author": "Qintel, LLC", + "license": "Apache 2.0", + "description": "Search Domains and Emails for exposed credentials", + "dataTypeList": ["mail","domain"], + "command": "Qintel/qintel_qwatch.py", + "baseConfig": "Qintel", + "configurationItems": [ + { + "name": "access_id", + "description": "Crosslink Access ID", + "type": "string", + "multi": false, + "required": true + }, + { + "name": "access_secret", + "description": "Crosslink Access Secret", + "type": "string", + "multi": false, + "required": true + }, + { + "name": "remote", + "description": "API URL (optional)", + "type": "string", + "multi": false, + "required": false + } + ] +} \ No newline at end of file diff --git a/analyzers/Qintel/qintel_qwatch.py b/analyzers/Qintel/qintel_qwatch.py new file mode 100755 index 000000000..c7bfb1299 --- /dev/null +++ b/analyzers/Qintel/qintel_qwatch.py @@ -0,0 +1,68 @@ +#!/usr/bin/env python3 +# encoding: utf-8 + +from cortexutils.analyzer import Analyzer + +from qintel_helper import search_qwatch + + +class QWatch(Analyzer): + + VERSION = '1.0' + + def __init__(self): + Analyzer.__init__(self) + self.client_id = self.get_param('config.access_id', None, + 'Missing Crosslink ID') + self.client_secret = self.get_param('config.access_secret', None, + 'Missing Crosslink Secret') + self.remote = self.get_param('config.remote', None) + + def _search(self, data): + + kwargs = { + 'remote': self.remote, + 'client_id': self.client_id, + 'client_secret': self.client_secret, + 'user_agent': f'cortex/{self.VERSION}', + 'params': { + 'meta[total]': True, + 'stats': True + } + } + + try: + return search_qwatch(data, self.data_type, 'exposures', **kwargs) + except RuntimeWarning: + pass + except Exception as e: + self.error(f'Qintel API: request failed, {str(e)}') + + def summary(self, raw): + taxonomies = [] + ns = 'Qintel' + level = 'info' + + count = self.res['meta']['total'] + taxonomies.append(self.build_taxonomy(level, ns, + 'CredentialCount', count)) + + return {'taxonomies': taxonomies} + + def run(self): + if self.data_type not in ['domain', 'mail']: + self.error('Unsupported data type') + + if self.data_type == 'mail': + self.data_type = 'email' + + data = self.getData() + self.res = self._search(data) + + self.report({ + 'Qintel_QWatch': self.res + }) + + +if __name__ == '__main__': + QWatch().run() diff --git a/thehive-templates/Qintel_QWatch/long.html b/thehive-templates/Qintel_QWatch/long.html new file mode 100644 index 000000000..a9af69722 --- /dev/null +++ b/thehive-templates/Qintel_QWatch/long.html @@ -0,0 +1,48 @@ + +
+
+ {{name}} +
+
+
+ {{content.Qintel_QWatch.meta.total}} exposed credentials in QWatch +

+
+
+ {{artifact.data | fang}} +

+ + + + + + + + + + + + + + + + + +
Login NamePasswordSource NameTimestamp
{{ r.attributes.login_name }}{{ r.attributes.password }}{{ r.attributes.source_name }}{{ r.attributes.timestamps[0].iso }}
+
+
+
+ + + +
+
+ {{artifact.data | fang}} +
+
+
+
{{name}}
+
{{content.errorMessage}}
+
+
+
\ No newline at end of file diff --git a/thehive-templates/Qintel_QWatch/short.html b/thehive-templates/Qintel_QWatch/short.html new file mode 100644 index 000000000..9fd48f9fa --- /dev/null +++ b/thehive-templates/Qintel_QWatch/short.html @@ -0,0 +1,3 @@ + + {{t.namespace}}:{{t.predicate}}="{{t.value}}" + \ No newline at end of file