From c2b37b5785f8a8002e66ed335260caea309a48c0 Mon Sep 17 00:00:00 2001 From: Petr Enkov Date: Tue, 6 Mar 2018 18:55:58 +0400 Subject: [PATCH 001/264] fix sending telegram cropped message --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 65adefe12..480ad2e7c 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1308,7 +1308,7 @@ def alert(self, matches): if len(matches) > 1: body += '\n----------------------------------------\n' if len(body) > 4095: - body = body[0:4000] + "\n⚠ *message was cropped according to telegram limits!* ⚠" + body = body[0:4000] + u"\n⚠ *message was cropped according to telegram limits!* ⚠" body += u' ```' headers = {'content-type': 'application/json'} From 68f6dadce14dd932dea5e51ca68e33e16a5712cf Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Thu, 8 Mar 2018 14:48:45 +0000 Subject: [PATCH 002/264] Naive attempt to allow rules to define their own writeback_index suffix --- elastalert/create_index.py | 3 +++ elastalert/elastalert.py | 12 +++++++----- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index ae859406c..1e594b5f7 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -170,6 +170,9 @@ def main(): # To avoid a race condition. TODO: replace this with a real check time.sleep(2) + # @TODO Also force ES v6 type indices + # @TODO Add alert specific index + # @TODO Add alert template for writeback_index suffixes if(elasticversion > 5): es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) es.indices.put_mapping(index=index+'_status', doc_type='elastalert_status', body=ess_mapping) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 597d821f0..3cca3aa36 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1369,13 +1369,14 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): alert_sent = True # Write the alert(s) to ES + writeback_index = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) agg_id = None for match in matches: alert_body = self.get_alert_body(match, rule, alert_sent, alert_time, alert_exception) # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, writeback_index) if res and not agg_id: agg_id = res['_id'] @@ -1399,9 +1400,9 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body): - writeback_index = self.writeback_index - if(self.is_atleastsix()): + def writeback(self, doc_type, body, index=None): + writeback_index = self.writeback_index if index is None else index + if self.is_atleastsix(): writeback_index = self.get_six_index(doc_type) # ES 2.0 - 2.3 does not support dots in field names. @@ -1636,7 +1637,8 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - res = self.writeback('elastalert', alert_body) + writeback_index = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) + res = self.writeback('elastalert', alert_body, writeback_index) # If new aggregation, save _id if res and not agg_id: From 366c7c30e72fd50c4d48ae9facfd3f779e7f1865 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 9 Mar 2018 14:18:35 +0000 Subject: [PATCH 003/264] Added writeback_index suffix template to create index --- elastalert/create_index.py | 5 ++++- elastalert/elastalert.py | 13 +++++++------ 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 1e594b5f7..9e7851062 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -174,11 +174,12 @@ def main(): # @TODO Add alert specific index # @TODO Add alert template for writeback_index suffixes if(elasticversion > 5): - es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) es.indices.put_mapping(index=index+'_status', doc_type='elastalert_status', body=ess_mapping) es.indices.put_mapping(index=index+'_silence', doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index+'_error', doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index+'_past', doc_type='past_elastalert', body=past_mapping) + es.indices.put_template(name='elastalert', body={'index_patterns': ['elastalert_*'], + 'aliases': {index: {}}, 'mappings': es_mapping}) print('New index %s created' % index) else: es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) @@ -186,6 +187,8 @@ def main(): es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index, doc_type='past_elastalert', body=past_mapping) + es.indices.put_template(name='elastalert', body={'template': 'elastalert_*', + 'aliases': {index: {}}, 'mappings': es_mapping}) print('New index %s created' % index) if old_index: diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 3cca3aa36..088286090 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1369,14 +1369,14 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): alert_sent = True # Write the alert(s) to ES - writeback_index = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) + writeback_index_suffix = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) agg_id = None for match in matches: alert_body = self.get_alert_body(match, rule, alert_sent, alert_time, alert_exception) # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body, writeback_index) + res = self.writeback('elastalert', alert_body, writeback_index_suffix) if res and not agg_id: agg_id = res['_id'] @@ -1400,8 +1400,9 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body, index=None): - writeback_index = self.writeback_index if index is None else index + def writeback(self, doc_type, body, index_suffix=None): + if index_suffix is not None: + writeback_index = self.get_six_index(doc_type) + '_' + index_suffix if self.is_atleastsix(): writeback_index = self.get_six_index(doc_type) @@ -1637,8 +1638,8 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - writeback_index = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) - res = self.writeback('elastalert', alert_body, writeback_index) + writeback_index_suffix = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) + res = self.writeback('elastalert', alert_body, writeback_index_suffix) # If new aggregation, save _id if res and not agg_id: From e075a6188a914bc3195ae3bac156725aaae542f1 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 9 Mar 2018 15:13:10 +0000 Subject: [PATCH 004/264] writeback_index now can have a single dynamic timestamp --- elastalert/elastalert.py | 23 ++++++++++++++++++----- 1 file changed, 18 insertions(+), 5 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 088286090..059b8c08c 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -199,6 +199,19 @@ def get_index(rule, starttime=None, endtime=None): else: return index + @staticmethod + def get_writeback_index(rule): + if 'writeback_index' not in rule: + return None + elif '%' not in rule['writeback_index']: + return rule['writeback_index'] + else: + index = rule['writeback_index'] + format_start = index.find('%') + format_end = index.rfind('%') + 2 + ts = datetime.datetime.utcnow().strftime(index[format_start:format_end]) + return index[:format_start] + ts + index[format_end:] + def get_six_index(self, doc_type): """ In ES6, you cannot have multiple _types per index, therefore we use self.writeback_index as the prefix for the actual @@ -1369,7 +1382,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): alert_sent = True # Write the alert(s) to ES - writeback_index_suffix = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) + writeback_index_suffix = self.get_writeback_index(rule) agg_id = None for match in matches: alert_body = self.get_alert_body(match, rule, alert_sent, alert_time, alert_exception) @@ -1401,9 +1414,10 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No return body def writeback(self, doc_type, body, index_suffix=None): + writeback_index = self.writeback_index if index_suffix is not None: - writeback_index = self.get_six_index(doc_type) + '_' + index_suffix - if self.is_atleastsix(): + writeback_index = writeback_index + '_' + index_suffix + elif self.is_atleastsix(): writeback_index = self.get_six_index(doc_type) # ES 2.0 - 2.3 does not support dots in field names. @@ -1638,8 +1652,7 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - writeback_index_suffix = None if not rule['writeback_index'] else self.get_index(rule['writeback_index']) - res = self.writeback('elastalert', alert_body, writeback_index_suffix) + res = self.writeback('elastalert', alert_body, self.get_writeback_index(rule)) # If new aggregation, save _id if res and not agg_id: From f1b419fc11831588eb1d6bfb580f0afd0d274ee8 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Mon, 12 Mar 2018 15:48:23 +0000 Subject: [PATCH 005/264] Added config file requirement of writeback_alias and used that when searching alerts Renamed writeback_index to writeback_suffix for rules --- config.yaml.example | 1 + elastalert/config.py | 3 +- elastalert/create_index.py | 21 +++++---- elastalert/elastalert.py | 96 ++++++++++++++++---------------------- elastalert/test_rule.py | 1 + tests/config_test.py | 3 +- tests/conftest.py | 1 + 7 files changed, 60 insertions(+), 66 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index beec38030..d3ebe69fd 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -57,6 +57,7 @@ es_port: 9200 # This can be a unmapped index, but it is recommended that you run # elastalert-create-index to set a mapping writeback_index: elastalert_status +writeback_alias: elastalert_alerts # If an alert fails for some reason, ElastAlert will retry # sending the alert until this time period has elapsed diff --git a/elastalert/config.py b/elastalert/config.py index 34dd4bd39..8627875de 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -29,7 +29,8 @@ rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) # Required global (config.yaml) and local (rule.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) +required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', + 'writeback_alias', 'buffer_time']) required_locals = frozenset(['alert', 'type', 'name', 'index']) # Settings that can be derived from ENV variables diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 9e7851062..4257506e4 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -32,6 +32,7 @@ def main(): parser.add_argument('--verify-certs', action='store_true', default=None, help='Verify TLS certificates') parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false', help='Do not verify TLS certificates') parser.add_argument('--index', help='Index name to create') + parser.add_argument('--alias', help='Alias name to create') parser.add_argument('--old-index', help='Old index name to copy') parser.add_argument('--send_get_body_as', default='GET', help='Method for querying Elasticsearch - POST, GET or source') parser.add_argument( @@ -74,6 +75,7 @@ def main(): client_cert = data.get('client_cert') client_key = data.get('client_key') index = args.index if args.index is not None else data.get('writeback_index') + alias = args.alias if args.alias is not None else data.get('writeback_alias') old_index = args.old_index if args.old_index is not None else None else: username = args.username if args.username else None @@ -100,6 +102,9 @@ def main(): index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ') if not index: index = 'elastalert_status' + alias = args.alias if args.alias is not None else raw_input('New alias name? (Default elastalert_alerts) ') + if not alias: + alias = 'elastalert_alias' old_index = (args.old_index if args.old_index is not None else raw_input('Name of existing index to copy? (Default None) ')) @@ -128,7 +133,7 @@ def main(): print("Elastic Version:" + esversion.split(".")[0]) elasticversion = int(esversion.split(".")[0]) - if(elasticversion > 5): + if elasticversion > 5: mapping = {'type': 'keyword'} else: mapping = {'index': 'not_analyzed', 'type': 'string'} @@ -158,7 +163,7 @@ def main(): print('Index ' + index + ' already exists. Skipping index creation.') return None - if (elasticversion > 5): + if elasticversion > 5: es.indices.create(index) es.indices.create(index+'_status') es.indices.create(index+'_silence') @@ -170,16 +175,15 @@ def main(): # To avoid a race condition. TODO: replace this with a real check time.sleep(2) - # @TODO Also force ES v6 type indices - # @TODO Add alert specific index - # @TODO Add alert template for writeback_index suffixes - if(elasticversion > 5): + if elasticversion > 5: + es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) es.indices.put_mapping(index=index+'_status', doc_type='elastalert_status', body=ess_mapping) es.indices.put_mapping(index=index+'_silence', doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index+'_error', doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index+'_past', doc_type='past_elastalert', body=past_mapping) + es.indices.put_alias(index=index, name=alias) es.indices.put_template(name='elastalert', body={'index_patterns': ['elastalert_*'], - 'aliases': {index: {}}, 'mappings': es_mapping}) + 'aliases': {alias: {}}, 'mappings': es_mapping}) print('New index %s created' % index) else: es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) @@ -187,8 +191,9 @@ def main(): es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index, doc_type='past_elastalert', body=past_mapping) + es.indices.put_alias(index=index, name=alias, body={'filter': {'term': {'_type': 'elastalert'}}}) es.indices.put_template(name='elastalert', body={'template': 'elastalert_*', - 'aliases': {index: {}}, 'mappings': es_mapping}) + 'aliases': {alias: {}}, 'mappings': es_mapping}) print('New index %s created' % index) if old_index: diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 059b8c08c..0d720b8f7 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -132,6 +132,7 @@ def __init__(self, args): self.scroll_keepalive = self.conf['scroll_keepalive'] self.rules = self.conf['rules'] self.writeback_index = self.conf['writeback_index'] + self.writeback_alias = self.conf['writeback_alias'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] self.old_query_limit = self.conf['old_query_limit'] @@ -199,32 +200,27 @@ def get_index(rule, starttime=None, endtime=None): else: return index - @staticmethod - def get_writeback_index(rule): - if 'writeback_index' not in rule: - return None - elif '%' not in rule['writeback_index']: - return rule['writeback_index'] - else: - index = rule['writeback_index'] - format_start = index.find('%') - format_end = index.rfind('%') + 2 - ts = datetime.datetime.utcnow().strftime(index[format_start:format_end]) - return index[:format_start] + ts + index[format_end:] - - def get_six_index(self, doc_type): - """ In ES6, you cannot have multiple _types per index, - therefore we use self.writeback_index as the prefix for the actual - index name, based on doc_type. """ + def get_writeback_index(self, doc_type, rule=None): writeback_index = self.writeback_index - if doc_type == 'silence': - writeback_index += '_silence' - elif doc_type == 'past_elastalert': - writeback_index += '_past' - elif doc_type == 'elastalert_status': - writeback_index += '_status' - elif doc_type == 'elastalert_error': - writeback_index += '_error' + if rule is None or 'writeback_suffix' not in rule: + if self.is_atleastsix(): + if doc_type == 'silence': + writeback_index += '_silence' + elif doc_type == 'past_elastalert': + writeback_index += '_past' + elif doc_type == 'elastalert_status': + writeback_index += '_status' + elif doc_type == 'elastalert_error': + writeback_index += '_error' + else: + suffix = rule['writeback_suffix'] + if '%' in rule['writeback_suffix']: + format_start = suffix.find('%') + format_end = suffix.rfind('%') + 2 + ts = datetime.datetime.utcnow().strftime(suffix[format_start:format_end]) + suffix = suffix[:format_start] + ts + suffix[format_end:] + writeback_index += '_' + suffix + return writeback_index @staticmethod @@ -657,13 +653,9 @@ def get_starttime(self, rule): query.update(sort) try: - if self.is_atleastsix(): - index = self.get_six_index('elastalert_status') - res = self.writeback_es.search(index=index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) - else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) + index = self.get_writeback_index('elastalert_status') + res = self.writeback_es.search(index=index, doc_type='elastalert_status', + size=1, body=query, _source_include=['endtime', 'rule_name']) if res['hits']['hits']: endtime = ts_to_dt(res['hits']['hits'][0]['_source']['endtime']) @@ -1081,7 +1073,7 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): ref = clock() while (clock() - ref) < timeout: try: - if self.writeback_es.indices.exists(self.writeback_index): + if self.writeback_es.indices.exists(self.writeback_alias): return except ConnectionError: pass @@ -1089,8 +1081,8 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): if self.writeback_es.ping(): logging.error( - 'Writeback index "%s" does not exist, did you run `elastalert-create-index`?', - self.writeback_index, + 'Writeback alias "%s" does not exist, did you run `elastalert-create-index`?', + self.writeback_alias, ) else: logging.error( @@ -1382,14 +1374,13 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): alert_sent = True # Write the alert(s) to ES - writeback_index_suffix = self.get_writeback_index(rule) agg_id = None for match in matches: alert_body = self.get_alert_body(match, rule, alert_sent, alert_time, alert_exception) # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body, writeback_index_suffix) + res = self.writeback('elastalert', alert_body, rule) if res and not agg_id: agg_id = res['_id'] @@ -1413,12 +1404,8 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body, index_suffix=None): - writeback_index = self.writeback_index - if index_suffix is not None: - writeback_index = writeback_index + '_' + index_suffix - elif self.is_atleastsix(): - writeback_index = self.get_six_index(doc_type) + def writeback(self, doc_type, body, rule=None): + writeback_index = self.get_writeback_index(doc_type, rule) # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: @@ -1463,7 +1450,7 @@ def find_recent_pending_alerts(self, time_limit): query = {'query': inner_query, 'filter': time_filter} query.update(sort) try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=1000) @@ -1477,6 +1464,7 @@ def send_pending_alerts(self): pending_alerts = self.find_recent_pending_alerts(self.alert_time_limit) for alert in pending_alerts: _id = alert['_id'] + _index = alert['_index'] alert = alert['_source'] try: rule_name = alert.pop('rule_name') @@ -1519,7 +1507,7 @@ def send_pending_alerts(self): # Delete it from the index try: - self.writeback_es.delete(index=self.writeback_index, + self.writeback_es.delete(index=_index, doc_type='elastalert', id=_id) except ElasticsearchException: # TODO: Give this a more relevant exception, try:except: is evil. @@ -1551,13 +1539,13 @@ def get_aggregated_matches(self, _id): query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}} matches = [] try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=self.max_aggregation) for match in res['hits']['hits']: matches.append(match['_source']) - self.writeback_es.delete(index=self.writeback_index, + self.writeback_es.delete(index=match['_index'], doc_type='elastalert', id=match['_id']) except (KeyError, ElasticsearchException) as e: @@ -1575,7 +1563,7 @@ def find_pending_aggregate_alert(self, rule, aggregation_key_value=None): query = {'query': {'bool': query}} query['sort'] = {'alert_time': {'order': 'desc'}} try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=1) @@ -1652,7 +1640,7 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - res = self.writeback('elastalert', alert_body, self.get_writeback_index(rule)) + res = self.writeback('elastalert', alert_body, rule) # If new aggregation, save _id if res and not agg_id: @@ -1717,13 +1705,9 @@ def is_silenced(self, rule_name): query.update(sort) try: - if(self.is_atleastsix()): - index = self.get_six_index('silence') - res = self.writeback_es.search(index=index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) - else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) + index = self.get_writeback_index('silence') + res = self.writeback_es.search(index=index, doc_type='silence', + size=1, body=query, _source_include=['until', 'exponent']) except ElasticsearchException as e: self.handle_error("Error while querying for alert silence status: %s" % (e), {'rule': rule_name}) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index e6be6004d..510e006bd 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -304,6 +304,7 @@ def load_conf(self, rules, args): 'es_host': 'localhost', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'max_query_size': 10000, 'alert_time_limit': datetime.timedelta(hours=24), 'old_query_limit': datetime.timedelta(weeks=1), diff --git a/tests/config_test.py b/tests/config_test.py index 640814013..035a32043 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -21,7 +21,8 @@ 'buffer_time': {'minutes': 10}, 'es_host': 'elasticsearch.test', 'es_port': 12345, - 'writeback_index': 'test_index'} + 'writeback_index': 'test_index', + 'writeback_alias': 'test_alias'} test_rule = {'es_host': 'test_host', 'es_port': 12345, diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..cd3671b7c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,6 +95,7 @@ def ea(): 'es_host': 'es', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'rules': rules, 'max_query_size': 10000, 'old_query_limit': datetime.timedelta(weeks=1), From 23172df4688909a73f22a9fe10734000af598cab Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Mon, 12 Mar 2018 16:24:06 +0000 Subject: [PATCH 006/264] writeback_alias added as required global config value writeback_suffix added to an optional rule value create_index.py added --alias option writeback_alias used when search for alerts more complex get_write_index used to generate appropriate writeback index --- config.yaml.example | 1 + elastalert/config.py | 3 +- elastalert/create_index.py | 17 ++++++-- elastalert/elastalert.py | 80 +++++++++++++++++++------------------- elastalert/test_rule.py | 1 + tests/config_test.py | 3 +- tests/conftest.py | 1 + 7 files changed, 61 insertions(+), 45 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index beec38030..d3ebe69fd 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -57,6 +57,7 @@ es_port: 9200 # This can be a unmapped index, but it is recommended that you run # elastalert-create-index to set a mapping writeback_index: elastalert_status +writeback_alias: elastalert_alerts # If an alert fails for some reason, ElastAlert will retry # sending the alert until this time period has elapsed diff --git a/elastalert/config.py b/elastalert/config.py index 34dd4bd39..8627875de 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -29,7 +29,8 @@ rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) # Required global (config.yaml) and local (rule.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) +required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', + 'writeback_alias', 'buffer_time']) required_locals = frozenset(['alert', 'type', 'name', 'index']) # Settings that can be derived from ENV variables diff --git a/elastalert/create_index.py b/elastalert/create_index.py index ae859406c..4257506e4 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -32,6 +32,7 @@ def main(): parser.add_argument('--verify-certs', action='store_true', default=None, help='Verify TLS certificates') parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false', help='Do not verify TLS certificates') parser.add_argument('--index', help='Index name to create') + parser.add_argument('--alias', help='Alias name to create') parser.add_argument('--old-index', help='Old index name to copy') parser.add_argument('--send_get_body_as', default='GET', help='Method for querying Elasticsearch - POST, GET or source') parser.add_argument( @@ -74,6 +75,7 @@ def main(): client_cert = data.get('client_cert') client_key = data.get('client_key') index = args.index if args.index is not None else data.get('writeback_index') + alias = args.alias if args.alias is not None else data.get('writeback_alias') old_index = args.old_index if args.old_index is not None else None else: username = args.username if args.username else None @@ -100,6 +102,9 @@ def main(): index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ') if not index: index = 'elastalert_status' + alias = args.alias if args.alias is not None else raw_input('New alias name? (Default elastalert_alerts) ') + if not alias: + alias = 'elastalert_alias' old_index = (args.old_index if args.old_index is not None else raw_input('Name of existing index to copy? (Default None) ')) @@ -128,7 +133,7 @@ def main(): print("Elastic Version:" + esversion.split(".")[0]) elasticversion = int(esversion.split(".")[0]) - if(elasticversion > 5): + if elasticversion > 5: mapping = {'type': 'keyword'} else: mapping = {'index': 'not_analyzed', 'type': 'string'} @@ -158,7 +163,7 @@ def main(): print('Index ' + index + ' already exists. Skipping index creation.') return None - if (elasticversion > 5): + if elasticversion > 5: es.indices.create(index) es.indices.create(index+'_status') es.indices.create(index+'_silence') @@ -170,12 +175,15 @@ def main(): # To avoid a race condition. TODO: replace this with a real check time.sleep(2) - if(elasticversion > 5): + if elasticversion > 5: es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) es.indices.put_mapping(index=index+'_status', doc_type='elastalert_status', body=ess_mapping) es.indices.put_mapping(index=index+'_silence', doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index+'_error', doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index+'_past', doc_type='past_elastalert', body=past_mapping) + es.indices.put_alias(index=index, name=alias) + es.indices.put_template(name='elastalert', body={'index_patterns': ['elastalert_*'], + 'aliases': {alias: {}}, 'mappings': es_mapping}) print('New index %s created' % index) else: es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) @@ -183,6 +191,9 @@ def main(): es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping) es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping) es.indices.put_mapping(index=index, doc_type='past_elastalert', body=past_mapping) + es.indices.put_alias(index=index, name=alias, body={'filter': {'term': {'_type': 'elastalert'}}}) + es.indices.put_template(name='elastalert', body={'template': 'elastalert_*', + 'aliases': {alias: {}}, 'mappings': es_mapping}) print('New index %s created' % index) if old_index: diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 597d821f0..0d720b8f7 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -132,6 +132,7 @@ def __init__(self, args): self.scroll_keepalive = self.conf['scroll_keepalive'] self.rules = self.conf['rules'] self.writeback_index = self.conf['writeback_index'] + self.writeback_alias = self.conf['writeback_alias'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] self.old_query_limit = self.conf['old_query_limit'] @@ -199,19 +200,27 @@ def get_index(rule, starttime=None, endtime=None): else: return index - def get_six_index(self, doc_type): - """ In ES6, you cannot have multiple _types per index, - therefore we use self.writeback_index as the prefix for the actual - index name, based on doc_type. """ + def get_writeback_index(self, doc_type, rule=None): writeback_index = self.writeback_index - if doc_type == 'silence': - writeback_index += '_silence' - elif doc_type == 'past_elastalert': - writeback_index += '_past' - elif doc_type == 'elastalert_status': - writeback_index += '_status' - elif doc_type == 'elastalert_error': - writeback_index += '_error' + if rule is None or 'writeback_suffix' not in rule: + if self.is_atleastsix(): + if doc_type == 'silence': + writeback_index += '_silence' + elif doc_type == 'past_elastalert': + writeback_index += '_past' + elif doc_type == 'elastalert_status': + writeback_index += '_status' + elif doc_type == 'elastalert_error': + writeback_index += '_error' + else: + suffix = rule['writeback_suffix'] + if '%' in rule['writeback_suffix']: + format_start = suffix.find('%') + format_end = suffix.rfind('%') + 2 + ts = datetime.datetime.utcnow().strftime(suffix[format_start:format_end]) + suffix = suffix[:format_start] + ts + suffix[format_end:] + writeback_index += '_' + suffix + return writeback_index @staticmethod @@ -644,13 +653,9 @@ def get_starttime(self, rule): query.update(sort) try: - if self.is_atleastsix(): - index = self.get_six_index('elastalert_status') - res = self.writeback_es.search(index=index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) - else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) + index = self.get_writeback_index('elastalert_status') + res = self.writeback_es.search(index=index, doc_type='elastalert_status', + size=1, body=query, _source_include=['endtime', 'rule_name']) if res['hits']['hits']: endtime = ts_to_dt(res['hits']['hits'][0]['_source']['endtime']) @@ -1068,7 +1073,7 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): ref = clock() while (clock() - ref) < timeout: try: - if self.writeback_es.indices.exists(self.writeback_index): + if self.writeback_es.indices.exists(self.writeback_alias): return except ConnectionError: pass @@ -1076,8 +1081,8 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): if self.writeback_es.ping(): logging.error( - 'Writeback index "%s" does not exist, did you run `elastalert-create-index`?', - self.writeback_index, + 'Writeback alias "%s" does not exist, did you run `elastalert-create-index`?', + self.writeback_alias, ) else: logging.error( @@ -1375,7 +1380,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, rule) if res and not agg_id: agg_id = res['_id'] @@ -1399,10 +1404,8 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body): - writeback_index = self.writeback_index - if(self.is_atleastsix()): - writeback_index = self.get_six_index(doc_type) + def writeback(self, doc_type, body, rule=None): + writeback_index = self.get_writeback_index(doc_type, rule) # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: @@ -1447,7 +1450,7 @@ def find_recent_pending_alerts(self, time_limit): query = {'query': inner_query, 'filter': time_filter} query.update(sort) try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=1000) @@ -1461,6 +1464,7 @@ def send_pending_alerts(self): pending_alerts = self.find_recent_pending_alerts(self.alert_time_limit) for alert in pending_alerts: _id = alert['_id'] + _index = alert['_index'] alert = alert['_source'] try: rule_name = alert.pop('rule_name') @@ -1503,7 +1507,7 @@ def send_pending_alerts(self): # Delete it from the index try: - self.writeback_es.delete(index=self.writeback_index, + self.writeback_es.delete(index=_index, doc_type='elastalert', id=_id) except ElasticsearchException: # TODO: Give this a more relevant exception, try:except: is evil. @@ -1535,13 +1539,13 @@ def get_aggregated_matches(self, _id): query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}} matches = [] try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=self.max_aggregation) for match in res['hits']['hits']: matches.append(match['_source']) - self.writeback_es.delete(index=self.writeback_index, + self.writeback_es.delete(index=match['_index'], doc_type='elastalert', id=match['_id']) except (KeyError, ElasticsearchException) as e: @@ -1559,7 +1563,7 @@ def find_pending_aggregate_alert(self, rule, aggregation_key_value=None): query = {'query': {'bool': query}} query['sort'] = {'alert_time': {'order': 'desc'}} try: - res = self.writeback_es.search(index=self.writeback_index, + res = self.writeback_es.search(index=self.writeback_alias, doc_type='elastalert', body=query, size=1) @@ -1636,7 +1640,7 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, rule) # If new aggregation, save _id if res and not agg_id: @@ -1701,13 +1705,9 @@ def is_silenced(self, rule_name): query.update(sort) try: - if(self.is_atleastsix()): - index = self.get_six_index('silence') - res = self.writeback_es.search(index=index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) - else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) + index = self.get_writeback_index('silence') + res = self.writeback_es.search(index=index, doc_type='silence', + size=1, body=query, _source_include=['until', 'exponent']) except ElasticsearchException as e: self.handle_error("Error while querying for alert silence status: %s" % (e), {'rule': rule_name}) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index e6be6004d..510e006bd 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -304,6 +304,7 @@ def load_conf(self, rules, args): 'es_host': 'localhost', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'max_query_size': 10000, 'alert_time_limit': datetime.timedelta(hours=24), 'old_query_limit': datetime.timedelta(weeks=1), diff --git a/tests/config_test.py b/tests/config_test.py index 640814013..035a32043 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -21,7 +21,8 @@ 'buffer_time': {'minutes': 10}, 'es_host': 'elasticsearch.test', 'es_port': 12345, - 'writeback_index': 'test_index'} + 'writeback_index': 'test_index', + 'writeback_alias': 'test_alias'} test_rule = {'es_host': 'test_host', 'es_port': 12345, diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..cd3671b7c 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -95,6 +95,7 @@ def ea(): 'es_host': 'es', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'rules': rules, 'max_query_size': 10000, 'old_query_limit': datetime.timedelta(weeks=1), From e94adc39921a22a164cfee8ebc1c64496e99dd05 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Mon, 26 Mar 2018 11:48:21 +0100 Subject: [PATCH 007/264] Updated tests to match changes to code base --- tests/base_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index fbb61ba9b..be2a299f9 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -260,7 +260,7 @@ def test_match_with_module_from_pending(ea): pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} # First call, return the pending alert, second, no associated aggregated alerts - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.send_pending_alerts() assert mod.process.call_count == 0 @@ -268,7 +268,7 @@ def test_match_with_module_from_pending(ea): # If aggregation is set, enhancement IS called pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.send_pending_alerts() @@ -346,9 +346,9 @@ def test_agg_matchtime(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call3}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call3}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call2}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: @@ -512,9 +512,9 @@ def test_agg_with_aggregation_key(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call2}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call2}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call3}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: @@ -1092,7 +1092,7 @@ def test_wait_until_responsive_timeout_index_does_not_exist(ea, capsys): # Ensure we get useful diagnostics. output, errors = capsys.readouterr() - assert 'Writeback index "wb" does not exist, did you run `elastalert-create-index`?' in errors + assert 'Writeback alias "wb_a" does not exist, did you run `elastalert-create-index`?' in errors # Slept until we passed the deadline. sleep.mock_calls == [ From 3f115f26681b48882a03afdf985c7204579ccb54 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Mon, 26 Mar 2018 11:49:09 +0100 Subject: [PATCH 008/264] Updated tests to match changes to code base --- tests/base_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/base_test.py b/tests/base_test.py index fbb61ba9b..be2a299f9 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -260,7 +260,7 @@ def test_match_with_module_from_pending(ea): pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} # First call, return the pending alert, second, no associated aggregated alerts - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.send_pending_alerts() assert mod.process.call_count == 0 @@ -268,7 +268,7 @@ def test_match_with_module_from_pending(ea): # If aggregation is set, enhancement IS called pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, {'hits': {'hits': []}}] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.send_pending_alerts() @@ -346,9 +346,9 @@ def test_agg_matchtime(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call3}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call3}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call2}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: @@ -512,9 +512,9 @@ def test_agg_with_aggregation_key(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call2}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}}, + ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call2}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call3}]}}, {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: @@ -1092,7 +1092,7 @@ def test_wait_until_responsive_timeout_index_does_not_exist(ea, capsys): # Ensure we get useful diagnostics. output, errors = capsys.readouterr() - assert 'Writeback index "wb" does not exist, did you run `elastalert-create-index`?' in errors + assert 'Writeback alias "wb_a" does not exist, did you run `elastalert-create-index`?' in errors # Slept until we passed the deadline. sleep.mock_calls == [ From 73d09befb5d93f4fd217b77be8e217b2091ec72a Mon Sep 17 00:00:00 2001 From: idealphase Date: Mon, 2 Apr 2018 04:21:31 +0700 Subject: [PATCH 009/264] Added LineNotify Alerter --- elastalert/alerts.py | 27 +++++++++++++++++++++++++++ elastalert/config.py | 3 ++- elastalert/schema.yaml | 3 +++ 3 files changed, 32 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 65adefe12..33f14cb50 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1560,3 +1560,30 @@ def get_info(self): return {'type': 'stride', 'stride_cloud_id': self.stride_cloud_id, 'stride_converstation_id': self.stride_converstation_id} +class LineNotifyAlerter(Alerter): + """ Created a Line Notify for each alert """ + required_option = frozenset(["linenotify_access_token"]) + + def __init__(self, rule): + super(LineNotifyAlerter, self).__init__(rule) + self.linenotify_access_token = self.rule["linenotify_access_token"] + + def alert(self, matches): + body = self.create_alert_body(matches) + # post to Line Notify + headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": "Bearer {}".format(self.linenotify_access_token) + } + payload = { + "message":body + } + try: + response = requests.post("https://notify-api.line.me/api/notify", data=payload, headers=headers) + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to Line Notify: %s" % e) + elastalert_logger.info("Alert sent to Line Notify") + + def get_info(self): + return {"type": "linenotify","linenotify_access_token": self.linenotify_access_token} diff --git a/elastalert/config.py b/elastalert/config.py index 701d84167..65e30ca1c 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -79,7 +79,8 @@ 'telegram': alerts.TelegramAlerter, 'gitter': alerts.GitterAlerter, 'servicenow': alerts.ServiceNowAlerter, - 'post': alerts.HTTPPostAlerter + 'post': alerts.HTTPPostAlerter, + 'linenotify': alerts.LineNotifyAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list # For example, jira goes before email so the ticket # will be added to the resulting email. diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index bffd8ee8b..4cfb1ef8b 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -275,3 +275,6 @@ properties: ### Simple simple_webhook_url: *arrayOfString simple_proxy: {type: string} + + ### LineNotify + linenotify_access_token: {type: string} From e44f1810fc54cb04d1a84ede11b324271244d2ca Mon Sep 17 00:00:00 2001 From: idealphase Date: Mon, 2 Apr 2018 04:22:42 +0700 Subject: [PATCH 010/264] Added LineNotify Alerter --- README.md | 1 + docs/source/ruletypes.rst | 10 ++++++++++ 2 files changed, 11 insertions(+) diff --git a/README.md b/README.md index b39128f4f..52b53e37a 100644 --- a/README.md +++ b/README.md @@ -53,6 +53,7 @@ Currently, we have built-in support for the following alert types: - Exotel - Twilio - Gitter +- Line Notify Additional rule types and alerts can be easily imported or written. diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index d9e89ee18..6dab1a4c9 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1740,3 +1740,13 @@ Example usage:: jira_priority: $priority$ jira_alert_owner: $owner$ + + +Line Notify +~~~~~~~~~~~ + +Line Notify will send notification to a Line application. The body of the notification is formatted the same as with other alerters. + +Required: + +``linenotify_access_token``: The access token that you got from https://notify-bot.line.me/my/ From aff2f05268a9c4279c0616a83106e9ebeefa31df Mon Sep 17 00:00:00 2001 From: idealphase Date: Mon, 2 Apr 2018 12:12:56 +0700 Subject: [PATCH 011/264] delete some whitespace , and add space after , : --- elastalert/alerts.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 33f14cb50..7ffabadf7 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1561,7 +1561,7 @@ def get_info(self): 'stride_cloud_id': self.stride_cloud_id, 'stride_converstation_id': self.stride_converstation_id} class LineNotifyAlerter(Alerter): - """ Created a Line Notify for each alert """ + """ Created a Line Notify for each alert """ required_option = frozenset(["linenotify_access_token"]) def __init__(self, rule): @@ -1576,7 +1576,7 @@ def alert(self, matches): "Authorization": "Bearer {}".format(self.linenotify_access_token) } payload = { - "message":body + "message": body } try: response = requests.post("https://notify-api.line.me/api/notify", data=payload, headers=headers) @@ -1586,4 +1586,4 @@ def alert(self, matches): elastalert_logger.info("Alert sent to Line Notify") def get_info(self): - return {"type": "linenotify","linenotify_access_token": self.linenotify_access_token} + return {"type": "linenotify", "linenotify_access_token": self.linenotify_access_token} From 61f00f082241a08cf9aab984e5ca00c718c976bc Mon Sep 17 00:00:00 2001 From: idealphase Date: Mon, 2 Apr 2018 12:30:08 +0700 Subject: [PATCH 012/264] delete some whitespace , and add 2 new blank lines --- elastalert/alerts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 7ffabadf7..74e75d75a 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1560,6 +1560,8 @@ def get_info(self): return {'type': 'stride', 'stride_cloud_id': self.stride_cloud_id, 'stride_converstation_id': self.stride_converstation_id} + + class LineNotifyAlerter(Alerter): """ Created a Line Notify for each alert """ required_option = frozenset(["linenotify_access_token"]) @@ -1576,7 +1578,7 @@ def alert(self, matches): "Authorization": "Bearer {}".format(self.linenotify_access_token) } payload = { - "message": body + "message": body } try: response = requests.post("https://notify-api.line.me/api/notify", data=payload, headers=headers) From e17d697c61912456f5517c8406e9ab77a4dbcee0 Mon Sep 17 00:00:00 2001 From: idealphase Date: Mon, 2 Apr 2018 12:37:36 +0700 Subject: [PATCH 013/264] delete some whitespace --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 74e75d75a..7b6cc963f 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1560,7 +1560,7 @@ def get_info(self): return {'type': 'stride', 'stride_cloud_id': self.stride_cloud_id, 'stride_converstation_id': self.stride_converstation_id} - + class LineNotifyAlerter(Alerter): """ Created a Line Notify for each alert """ From 37d5a0d5dddc97c23848c6342063b88d55227034 Mon Sep 17 00:00:00 2001 From: Andre Caron Date: Tue, 3 Apr 2018 12:12:08 -0400 Subject: [PATCH 014/264] Relax requirement on `doc_type` option in `metric_aggregation` rules Presence of the `doc_type` option in metric aggregation rules is enforced, but the code doesn't seem to use it and metric aggregation rules work fine in practice when you remove this requirement. --- elastalert/ruletypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index e7b218b17..f4160181a 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -949,7 +949,7 @@ def check_matches(self, timestamp, query_key, aggregation_data): class MetricAggregationRule(BaseAggregationRule): """ A rule that matches when there is a low number of events given a timeframe. """ - required_options = frozenset(['metric_agg_key', 'metric_agg_type', 'doc_type']) + required_options = frozenset(['metric_agg_key', 'metric_agg_type']) allowed_aggregations = frozenset(['min', 'max', 'avg', 'sum', 'cardinality', 'value_count']) def __init__(self, *args): From 3fbf82f91dee57cb97cb0cbb65e654f831b34ef1 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Tue, 17 Apr 2018 11:44:13 +0100 Subject: [PATCH 015/264] Added template check. Deletes if templates exists and index does not --- elastalert/create_index.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 4257506e4..edef17b37 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -162,6 +162,9 @@ def main(): if es_index.exists(index): print('Index ' + index + ' already exists. Skipping index creation.') return None + elif es_index.exists_template(index): + print('Template ' + index + ' already exists. Deleting in preparation for creating indexes.') + es_index.delete_template(index) if elasticversion > 5: es.indices.create(index) From 000ce74330dd890ca662897d16b439817f49bfad Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Wed, 11 Jul 2018 10:48:54 +0100 Subject: [PATCH 016/264] Extracting out rule loading so that rules can be stored in others other than on disk --- elastalert/config.py | 464 ++-------------------------------- elastalert/elastalert.py | 33 ++- elastalert/ruleloaders.py | 505 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 536 insertions(+), 466 deletions(-) create mode 100644 elastalert/ruleloaders.py diff --git a/elastalert/config.py b/elastalert/config.py index c6efb3ad2..0e21fd42b 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -1,36 +1,15 @@ # -*- coding: utf-8 -*- -import copy import datetime -import hashlib -import logging -import os import sys -import alerts -import enhancements -import jsonschema -import ruletypes -import yaml -import yaml.scanner +import ruleloaders from envparse import Env -from opsgenie import OpsGenieAlerter from staticconf.loader import yaml_loader -from util import dt_to_ts -from util import dt_to_ts_with_format -from util import dt_to_unix -from util import dt_to_unixms from util import EAException -from util import ts_to_dt -from util import ts_to_dt_with_format -from util import unix_to_dt -from util import unixms_to_dt -# schema for rule yaml -rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) -# Required global (config.yaml) and local (rule.yaml) configuration options +# Required global (config.yaml) configuration options required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) -required_locals = frozenset(['alert', 'type', 'name', 'index']) # Settings that can be derived from ENV variables env_settings = {'ES_USE_SSL': 'use_ssl', @@ -42,55 +21,10 @@ env = Env(ES_USE_SSL=bool) -# import rule dependency -import_rules = {} - -# Used to map the names of rules to their classes -rules_mapping = { - 'frequency': ruletypes.FrequencyRule, - 'any': ruletypes.AnyRule, - 'spike': ruletypes.SpikeRule, - 'blacklist': ruletypes.BlacklistRule, - 'whitelist': ruletypes.WhitelistRule, - 'change': ruletypes.ChangeRule, - 'flatline': ruletypes.FlatlineRule, - 'new_term': ruletypes.NewTermsRule, - 'cardinality': ruletypes.CardinalityRule, - 'metric_aggregation': ruletypes.MetricAggregationRule, - 'percentage_match': ruletypes.PercentageMatchRule, -} - -# Used to map names of alerts to their classes -alerts_mapping = { - 'email': alerts.EmailAlerter, - 'jira': alerts.JiraAlerter, - 'opsgenie': OpsGenieAlerter, - 'stomp': alerts.StompAlerter, - 'debug': alerts.DebugAlerter, - 'command': alerts.CommandAlerter, - 'sns': alerts.SnsAlerter, - 'hipchat': alerts.HipChatAlerter, - 'stride': alerts.StrideAlerter, - 'ms_teams': alerts.MsTeamsAlerter, - 'slack': alerts.SlackAlerter, - 'pagerduty': alerts.PagerDutyAlerter, - 'exotel': alerts.ExotelAlerter, - 'twilio': alerts.TwilioAlerter, - 'victorops': alerts.VictorOpsAlerter, - 'telegram': alerts.TelegramAlerter, - 'gitter': alerts.GitterAlerter, - 'servicenow': alerts.ServiceNowAlerter, - 'alerta': alerts.AlertaAlerter, - 'post': alerts.HTTPPostAlerter +# Used to map the names of rule loaders to their classes +loader_mapping = { + 'file': ruleloaders.FileRulesLoader } -# A partial ordering of alert types. Relative order will be preserved in the resulting alerts list -# For example, jira goes before email so the ticket # will be added to the resulting email. -alerts_order = { - 'jira': 0, - 'email': 1 -} - -base_config = {} def get_module(module_name): @@ -106,337 +40,15 @@ def get_module(module_name): return module -def load_configuration(filename, conf, args=None): - """ Load a yaml rule file and fill in the relevant fields with objects. - - :param filename: The name of a rule configuration file. - :param conf: The global configuration dictionary, used for populating defaults. - :return: The rule configuration, a dictionary. - """ - rule = load_rule_yaml(filename) - load_options(rule, conf, filename, args) - load_modules(rule, args) - return rule - - -def load_rule_yaml(filename): - rule = { - 'rule_file': filename, - } - - import_rules.pop(filename, None) # clear `filename` dependency - while True: - try: - loaded = yaml_loader(filename) - except yaml.scanner.ScannerError as e: - raise EAException('Could not parse file %s: %s' % (filename, e)) - - # Special case for merging filters - if both files specify a filter merge (AND) them - if 'filter' in rule and 'filter' in loaded: - rule['filter'] = loaded['filter'] + rule['filter'] - - loaded.update(rule) - rule = loaded - if 'import' in rule: - # Find the path of the next file. - if os.path.isabs(rule['import']): - import_filename = rule['import'] - else: - import_filename = os.path.join(os.path.dirname(filename), rule['import']) - # set dependencies - rules = import_rules.get(filename, []) - rules.append(import_filename) - import_rules[filename] = rules - filename = import_filename - del(rule['import']) # or we could go on forever! - else: - break - - return rule - - -def load_options(rule, conf, filename, args=None): - """ Converts time objects, sets defaults, and validates some settings. - - :param rule: A dictionary of parsed YAML from a rule config file. - :param conf: The global configuration dictionary, used for populating defaults. - """ - adjust_deprecated_values(rule) - - try: - rule_schema.validate(rule) - except jsonschema.ValidationError as e: - raise EAException("Invalid Rule file: %s\n%s" % (filename, e)) - - try: - # Set all time based parameters - if 'timeframe' in rule: - rule['timeframe'] = datetime.timedelta(**rule['timeframe']) - if 'realert' in rule: - rule['realert'] = datetime.timedelta(**rule['realert']) - else: - if 'aggregation' in rule: - rule['realert'] = datetime.timedelta(minutes=0) - else: - rule['realert'] = datetime.timedelta(minutes=1) - if 'aggregation' in rule and not rule['aggregation'].get('schedule'): - rule['aggregation'] = datetime.timedelta(**rule['aggregation']) - if 'query_delay' in rule: - rule['query_delay'] = datetime.timedelta(**rule['query_delay']) - if 'buffer_time' in rule: - rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) - if 'bucket_interval' in rule: - rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) - if 'exponential_realert' in rule: - rule['exponential_realert'] = datetime.timedelta(**rule['exponential_realert']) - if 'kibana4_start_timedelta' in rule: - rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) - if 'kibana4_end_timedelta' in rule: - rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) - except (KeyError, TypeError) as e: - raise EAException('Invalid time format used: %s' % (e)) - - # Set defaults, copy defaults from config.yaml - for key, val in base_config.items(): - rule.setdefault(key, val) - rule.setdefault('name', os.path.splitext(filename)[0]) - rule.setdefault('realert', datetime.timedelta(seconds=0)) - rule.setdefault('aggregation', datetime.timedelta(seconds=0)) - rule.setdefault('query_delay', datetime.timedelta(seconds=0)) - rule.setdefault('timestamp_field', '@timestamp') - rule.setdefault('filter', []) - rule.setdefault('timestamp_type', 'iso') - rule.setdefault('timestamp_format', '%Y-%m-%dT%H:%M:%SZ') - rule.setdefault('_source_enabled', True) - rule.setdefault('use_local_time', True) - rule.setdefault('description', "") - - # Set timestamp_type conversion function, used when generating queries and processing hits - rule['timestamp_type'] = rule['timestamp_type'].strip().lower() - if rule['timestamp_type'] == 'iso': - rule['ts_to_dt'] = ts_to_dt - rule['dt_to_ts'] = dt_to_ts - elif rule['timestamp_type'] == 'unix': - rule['ts_to_dt'] = unix_to_dt - rule['dt_to_ts'] = dt_to_unix - elif rule['timestamp_type'] == 'unix_ms': - rule['ts_to_dt'] = unixms_to_dt - rule['dt_to_ts'] = dt_to_unixms - elif rule['timestamp_type'] == 'custom': - def _ts_to_dt_with_format(ts): - return ts_to_dt_with_format(ts, ts_format=rule['timestamp_format']) - - def _dt_to_ts_with_format(dt): - ts = dt_to_ts_with_format(dt, ts_format=rule['timestamp_format']) - if 'timestamp_format_expr' in rule: - # eval expression passing 'ts' and 'dt' - return eval(rule['timestamp_format_expr'], {'ts': ts, 'dt': dt}) - else: - return ts - - rule['ts_to_dt'] = _ts_to_dt_with_format - rule['dt_to_ts'] = _dt_to_ts_with_format - else: - raise EAException('timestamp_type must be one of iso, unix, or unix_ms') - - # Add support for client ssl certificate auth - if 'verify_certs' in conf: - rule.setdefault('verify_certs', conf.get('verify_certs')) - rule.setdefault('ca_certs', conf.get('ca_certs')) - rule.setdefault('client_cert', conf.get('client_cert')) - rule.setdefault('client_key', conf.get('client_key')) - - # Set HipChat options from global config - rule.setdefault('hipchat_msg_color', 'red') - rule.setdefault('hipchat_domain', 'api.hipchat.com') - rule.setdefault('hipchat_notify', True) - rule.setdefault('hipchat_from', '') - rule.setdefault('hipchat_ignore_ssl_errors', False) - - # Make sure we have required options - if required_locals - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(required_locals - frozenset(rule.keys())))) - - if 'include' in rule and type(rule['include']) != list: - raise EAException('include option must be a list') - - if isinstance(rule.get('query_key'), list): - rule['compound_query_key'] = rule['query_key'] - rule['query_key'] = ','.join(rule['query_key']) - - if isinstance(rule.get('aggregation_key'), list): - rule['compound_aggregation_key'] = rule['aggregation_key'] - rule['aggregation_key'] = ','.join(rule['aggregation_key']) - - if isinstance(rule.get('compare_key'), list): - rule['compound_compare_key'] = rule['compare_key'] - rule['compare_key'] = ','.join(rule['compare_key']) - elif 'compare_key' in rule: - rule['compound_compare_key'] = [rule['compare_key']] - # Add QK, CK and timestamp to include - include = rule.get('include', ['*']) - if 'query_key' in rule: - include.append(rule['query_key']) - if 'compound_query_key' in rule: - include += rule['compound_query_key'] - if 'compound_aggregation_key' in rule: - include += rule['compound_aggregation_key'] - if 'compare_key' in rule: - include.append(rule['compare_key']) - if 'compound_compare_key' in rule: - include += rule['compound_compare_key'] - if 'top_count_keys' in rule: - include += rule['top_count_keys'] - include.append(rule['timestamp_field']) - rule['include'] = list(set(include)) - - # Check that generate_kibana_url is compatible with the filters - if rule.get('generate_kibana_link'): - for es_filter in rule.get('filter'): - if es_filter: - if 'not' in es_filter: - es_filter = es_filter['not'] - if 'query' in es_filter: - es_filter = es_filter['query'] - if es_filter.keys()[0] not in ('term', 'query_string', 'range'): - raise EAException('generate_kibana_link is incompatible with filters other than term, query_string and range. ' - 'Consider creating a dashboard and using use_kibana_dashboard instead.') - - # Check that doc_type is provided if use_count/terms_query - if rule.get('use_count_query') or rule.get('use_terms_query'): - if 'doc_type' not in rule: - raise EAException('doc_type must be specified.') - - # Check that query_key is set if use_terms_query - if rule.get('use_terms_query'): - if 'query_key' not in rule: - raise EAException('query_key must be specified with use_terms_query') - - # Warn if use_strf_index is used with %y, %M or %D - # (%y = short year, %M = minutes, %D = full date) - if rule.get('use_strftime_index'): - for token in ['%y', '%M', '%D']: - if token in rule.get('index'): - logging.warning('Did you mean to use %s in the index? ' - 'The index will be formatted like %s' % (token, - datetime.datetime.now().strftime(rule.get('index')))) - - if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): - raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') - - -def load_modules(rule, args=None): - """ Loads things that could be modules. Enhancements, alerts and rule type. """ - # Set match enhancements - match_enhancements = [] - for enhancement_name in rule.get('match_enhancements', []): - if enhancement_name in dir(enhancements): - enhancement = getattr(enhancements, enhancement_name) - else: - enhancement = get_module(enhancement_name) - if not issubclass(enhancement, enhancements.BaseEnhancement): - raise EAException("Enhancement module %s not a subclass of BaseEnhancement" % (enhancement_name)) - match_enhancements.append(enhancement(rule)) - rule['match_enhancements'] = match_enhancements - - # Convert rule type into RuleType object - if rule['type'] in rules_mapping: - rule['type'] = rules_mapping[rule['type']] - else: - rule['type'] = get_module(rule['type']) - if not issubclass(rule['type'], ruletypes.RuleType): - raise EAException('Rule module %s is not a subclass of RuleType' % (rule['type'])) - - # Make sure we have required alert and type options - reqs = rule['type'].required_options - - if reqs - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(rule.keys())))) - # Instantiate rule - try: - rule['type'] = rule['type'](rule, args) - except (KeyError, EAException) as e: - raise EAException('Error initializing rule %s: %s' % (rule['name'], e)), None, sys.exc_info()[2] - # Instantiate alerts only if we're not in debug mode - # In debug mode alerts are not actually sent so don't bother instantiating them - if not args or not args.debug: - rule['alert'] = load_alerts(rule, alert_field=rule['alert']) - - -def isyaml(filename): - return filename.endswith('.yaml') or filename.endswith('.yml') - - -def get_file_paths(conf, use_rule=None): - # Passing a filename directly can bypass rules_folder and .yaml checks - if use_rule and os.path.isfile(use_rule): - return [use_rule] - rule_folder = conf['rules_folder'] - rule_files = [] - if conf['scan_subdirectories']: - for root, folders, files in os.walk(rule_folder): - for filename in files: - if use_rule and use_rule != filename: - continue - if isyaml(filename): - rule_files.append(os.path.join(root, filename)) - else: - for filename in os.listdir(rule_folder): - fullpath = os.path.join(rule_folder, filename) - if os.path.isfile(fullpath) and isyaml(filename): - rule_files.append(fullpath) - return rule_files - - -def load_alerts(rule, alert_field): - def normalize_config(alert): - """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. - This function normalizes them both to the latter format. """ - if isinstance(alert, basestring): - return alert, rule - elif isinstance(alert, dict): - name, config = iter(alert.items()).next() - config_copy = copy.copy(rule) - config_copy.update(config) # warning, this (intentionally) mutates the rule dict - return name, config_copy - else: - raise EAException() - - def create_alert(alert, alert_config): - alert_class = alerts_mapping.get(alert) or get_module(alert) - if not issubclass(alert_class, alerts.Alerter): - raise EAException('Alert module %s is not a subclass of Alerter' % (alert)) - missing_options = (rule['type'].required_options | alert_class.required_options) - frozenset(alert_config or []) - if missing_options: - raise EAException('Missing required option(s): %s' % (', '.join(missing_options))) - return alert_class(alert_config) - - try: - if type(alert_field) != list: - alert_field = [alert_field] - - alert_field = [normalize_config(x) for x in alert_field] - alert_field = sorted(alert_field, key=lambda (a, b): alerts_order.get(a, 1)) - # Convert all alerts into Alerter objects - alert_field = [create_alert(a, b) for a, b in alert_field] - - except (KeyError, EAException) as e: - raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)), None, sys.exc_info()[2] - - return alert_field - - -def load_rules(args): +def load_conf(args): """ Creates a conf dictionary for ElastAlerter. Loads the global - config file and then each rule found in rules_folder. + config file and then each rule found in rules_folder. - :param args: The parsed arguments to ElastAlert - :return: The global configuration, a dictionary. - """ - names = [] + :param args: The parsed arguments to ElastAlert + :return: The global configuration, a dictionary. + """ filename = args.config conf = yaml_loader(filename) - use_rule = args.rule for env_var, conf_var in env_settings.items(): val = env(env_var, None) @@ -451,6 +63,7 @@ def load_rules(args): conf.setdefault('scroll_keepalive', '30s') conf.setdefault('disable_rules_on_error', True) conf.setdefault('scan_subdirectories', True) + conf.setdefault('rules_loader', 'file') # Convert run_every, buffer_time into a timedelta object try: @@ -465,56 +78,11 @@ def load_rules(args): else: conf['old_query_limit'] = datetime.timedelta(weeks=1) except (KeyError, TypeError) as e: - raise EAException('Invalid time format used: %s' % (e)) + raise EAException('Invalid time format used: %s' % e) - global base_config - base_config = copy.deepcopy(conf) + # Initialise the rule loader and load each rule configuration + rules_loader_class = loader_mapping.get(conf['rules_loader']) or get_module(conf['rules_loader']) + rules_loader = rules_loader_class(conf) + conf['rules_loader'] = rules_loader - # Load each rule configuration file - rules = [] - rule_files = get_file_paths(conf, use_rule) - for rule_file in rule_files: - try: - rule = load_configuration(rule_file, conf, args) - # By setting "is_enabled: False" in rule file, a rule is easily disabled - if 'is_enabled' in rule and not rule['is_enabled']: - continue - if rule['name'] in names: - raise EAException('Duplicate rule named %s' % (rule['name'])) - except EAException as e: - raise EAException('Error loading file %s: %s' % (rule_file, e)) - - rules.append(rule) - names.append(rule['name']) - - conf['rules'] = rules return conf - - -def get_rule_hashes(conf, use_rule=None): - rule_files = get_file_paths(conf, use_rule) - rule_mod_times = {} - for rule_file in rule_files: - rule_mod_times[rule_file] = get_rulefile_hash(rule_file) - return rule_mod_times - - -def get_rulefile_hash(rule_file): - rulefile_hash = '' - if os.path.exists(rule_file): - with open(rule_file) as fh: - rulefile_hash = hashlib.sha1(fh.read()).digest() - for import_rule_file in import_rules.get(rule_file, []): - rulefile_hash += get_rulefile_hash(import_rule_file) - return rulefile_hash - - -def adjust_deprecated_values(rule): - # From rename of simple HTTP alerter - if rule.get('type') == 'simple': - rule['type'] = 'post' - if 'simple_proxy' in rule: - rule['http_post_proxy'] = rule['simple_proxy'] - if 'simple_webhook_url' in rule: - rule['http_post_url'] = rule['simple_webhook_url'] - logging.warning('"simple" alerter has been renamed "post" and comptability may be removed in a future release.') diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index ed389cfd6..abcc027bf 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -17,11 +17,8 @@ import dateutil.tz import kibana -import yaml from alerts import DebugAlerter -from config import get_rule_hashes -from config import load_configuration -from config import load_rules +from config import load_conf from croniter import croniter from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException @@ -127,10 +124,10 @@ def __init__(self, args): tracer.setLevel(logging.INFO) tracer.addHandler(logging.FileHandler(self.args.es_debug_trace)) - self.conf = load_rules(self.args) + self.conf = load_conf(self.args) self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] - self.rules = self.conf['rules'] + self.rules = self.conf['rules_loader'].load_all(self.conf, self.args) self.writeback_index = self.conf['writeback_index'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] @@ -147,7 +144,7 @@ def __init__(self, args): self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} - self.rule_hashes = get_rule_hashes(self.conf, self.args.rule) + self.rule_hashes = self.conf['rules_loader'].get_hashes(self.conf, self.args) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) @@ -967,9 +964,10 @@ def modify_rule_for_ES5(new_rule): new_rule['filter'] = new_filters def load_rule_changes(self): - ''' Using the modification times of rule config files, syncs the running rules - to match the files in rules_folder by removing, adding or reloading rules. ''' - new_rule_hashes = get_rule_hashes(self.conf, self.args.rule) + """ Using the modification times of rule config files, syncs the running rules + to match the files in rules_folder by removing, adding or reloading rules. """ + rules_loader = self.conf['rules_loader'] + new_rule_hashes = rules_loader.get_hashes(self.conf, self.args.rule) # Check each current rule for changes for rule_file, hash_value in self.rule_hashes.iteritems(): @@ -981,7 +979,7 @@ def load_rule_changes(self): if hash_value != new_rule_hashes[rule_file]: # Rule file was changed, reload rule try: - new_rule = load_configuration(rule_file, self.conf) + new_rule = rules_loader.load_configuration(rule_file, self.conf) if 'is_enabled' in new_rule and not new_rule['is_enabled']: elastalert_logger.info('Rule file %s is now disabled.' % (rule_file)) # Remove this rule if it's been disabled @@ -991,12 +989,11 @@ def load_rule_changes(self): message = 'Could not load rule %s: %s' % (rule_file, e) self.handle_error(message) # Want to send email to address specified in the rule. Try and load the YAML to find it. - with open(rule_file) as f: - try: - rule_yaml = yaml.load(f) - except yaml.scanner.ScannerError: - self.send_notification_email(exception=e) - continue + try: + rule_yaml = rules_loader.load_yaml(rule_file) + except EAException: + self.send_notification_email(exception=e) + continue self.send_notification_email(exception=e, rule=rule_yaml) continue @@ -1019,7 +1016,7 @@ def load_rule_changes(self): if not self.args.rule: for rule_file in set(new_rule_hashes.keys()) - set(self.rule_hashes.keys()): try: - new_rule = load_configuration(rule_file, self.conf) + new_rule = rules_loader.load_configuration(rule_file, self.conf) if 'is_enabled' in new_rule and not new_rule['is_enabled']: continue if new_rule['name'] in [rule['name'] for rule in self.rules]: diff --git a/elastalert/ruleloaders.py b/elastalert/ruleloaders.py new file mode 100644 index 000000000..fccab7605 --- /dev/null +++ b/elastalert/ruleloaders.py @@ -0,0 +1,505 @@ +# -*- coding: utf-8 -*- +import copy +import datetime +import hashlib +import logging +import os +import sys + +import alerts +import enhancements +import jsonschema +import ruletypes +import yaml +import yaml.scanner +from opsgenie import OpsGenieAlerter +from staticconf.loader import yaml_loader +from config import get_module +from util import dt_to_ts +from util import dt_to_ts_with_format +from util import dt_to_unix +from util import dt_to_unixms +from util import EAException +from util import ts_to_dt +from util import ts_to_dt_with_format +from util import unix_to_dt +from util import unixms_to_dt + + +class RulesLoader(object): + # import rule dependency + import_rules = {} + + # Required local (rule.yaml) configuration options + required_locals = frozenset(['alert', 'type', 'name', 'index']) + + # Used to map the names of rules to their classes + rules_mapping = { + 'frequency': ruletypes.FrequencyRule, + 'any': ruletypes.AnyRule, + 'spike': ruletypes.SpikeRule, + 'blacklist': ruletypes.BlacklistRule, + 'whitelist': ruletypes.WhitelistRule, + 'change': ruletypes.ChangeRule, + 'flatline': ruletypes.FlatlineRule, + 'new_term': ruletypes.NewTermsRule, + 'cardinality': ruletypes.CardinalityRule, + 'metric_aggregation': ruletypes.MetricAggregationRule, + 'percentage_match': ruletypes.PercentageMatchRule, + } + + # Used to map names of alerts to their classes + alerts_mapping = { + 'email': alerts.EmailAlerter, + 'jira': alerts.JiraAlerter, + 'opsgenie': OpsGenieAlerter, + 'stomp': alerts.StompAlerter, + 'debug': alerts.DebugAlerter, + 'command': alerts.CommandAlerter, + 'sns': alerts.SnsAlerter, + 'hipchat': alerts.HipChatAlerter, + 'stride': alerts.StrideAlerter, + 'ms_teams': alerts.MsTeamsAlerter, + 'slack': alerts.SlackAlerter, + 'pagerduty': alerts.PagerDutyAlerter, + 'exotel': alerts.ExotelAlerter, + 'twilio': alerts.TwilioAlerter, + 'victorops': alerts.VictorOpsAlerter, + 'telegram': alerts.TelegramAlerter, + 'gitter': alerts.GitterAlerter, + 'servicenow': alerts.ServiceNowAlerter, + 'alerta': alerts.AlertaAlerter, + 'post': alerts.HTTPPostAlerter + } + + # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list + # For example, jira goes before email so the ticket # will be added to the resulting email. + alerts_order = { + 'jira': 0, + 'email': 1 + } + + base_config = {} + + def __init__(self, conf): + # schema for rule yaml + self.rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) + + self.base_config = copy.deepcopy(conf) + + def load_all(self, conf, args): + """ + Load all the rules and return them. + :param conf: Configuration dict + :param args: Arguments dict + :return: List of rules + """ + raise NotImplementedError() + + def get_hashes(self, conf, use_rule=None): + """ + Get hashes of the rules. + :param conf: Configuration dict + :param use_rule: Limit to only specified rule + :return: Dict of rule name to hash + """ + raise NotImplementedError() + + def get_yaml(self, filename): + """ + Get and parse the yaml of the specified rule. + :param filename: Rule to get the yaml + :return: Rule YAML dict + """ + raise NotImplementedError() + + def get_import_rule(self, rule): + """ + :param rule: Rule dict + :return: rule name that will all `get_yaml` to retrieve the yaml of the rule + """ + raise NotImplementedError() + + def load_yaml(self, filename): + """ + Load the rule including all dependency rules. + :param filename: Rule to load + :return: Loaded rule dict + """ + rule = { + 'rule_file': filename, + } + + self.import_rules.pop(filename, None) # clear `filename` dependency + while True: + loaded = self.get_yaml(filename) + + # Special case for merging filters - if both files specify a filter merge (AND) them + if 'filter' in rule and 'filter' in loaded: + rule['filter'] = loaded['filter'] + rule['filter'] + + loaded.update(rule) + rule = loaded + if 'import' in rule: + # Find the path of the next file. + import_filename = self.get_import_rule(rule) + # set dependencies + rules = self.import_rules.get(filename, []) + rules.append(import_filename) + self.import_rules[filename] = rules + filename = import_filename + del (rule['import']) # or we could go on forever! + else: + break + + return rule + + def load_options(self, rule, conf, filename, args=None): + """ Converts time objects, sets defaults, and validates some settings. + + :param rule: A dictionary of parsed YAML from a rule config file. + :param conf: The global configuration dictionary, used for populating defaults. + :param filename: Name of the rule + :param args: Arguments + """ + self.adjust_deprecated_values(rule) + + try: + self.rule_schema.validate(rule) + except jsonschema.ValidationError as e: + raise EAException("Invalid Rule file: %s\n%s" % (filename, e)) + + try: + # Set all time based parameters + if 'timeframe' in rule: + rule['timeframe'] = datetime.timedelta(**rule['timeframe']) + if 'realert' in rule: + rule['realert'] = datetime.timedelta(**rule['realert']) + else: + if 'aggregation' in rule: + rule['realert'] = datetime.timedelta(minutes=0) + else: + rule['realert'] = datetime.timedelta(minutes=1) + if 'aggregation' in rule and not rule['aggregation'].get('schedule'): + rule['aggregation'] = datetime.timedelta(**rule['aggregation']) + if 'query_delay' in rule: + rule['query_delay'] = datetime.timedelta(**rule['query_delay']) + if 'buffer_time' in rule: + rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) + if 'bucket_interval' in rule: + rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) + if 'exponential_realert' in rule: + rule['exponential_realert'] = datetime.timedelta(**rule['exponential_realert']) + if 'kibana4_start_timedelta' in rule: + rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) + if 'kibana4_end_timedelta' in rule: + rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) + except (KeyError, TypeError) as e: + raise EAException('Invalid time format used: %s' % e) + + # Set defaults, copy defaults from config.yaml + for key, val in self.base_config.items(): + rule.setdefault(key, val) + rule.setdefault('name', os.path.splitext(filename)[0]) + rule.setdefault('realert', datetime.timedelta(seconds=0)) + rule.setdefault('aggregation', datetime.timedelta(seconds=0)) + rule.setdefault('query_delay', datetime.timedelta(seconds=0)) + rule.setdefault('timestamp_field', '@timestamp') + rule.setdefault('filter', []) + rule.setdefault('timestamp_type', 'iso') + rule.setdefault('timestamp_format', '%Y-%m-%dT%H:%M:%SZ') + rule.setdefault('_source_enabled', True) + rule.setdefault('use_local_time', True) + rule.setdefault('description', "") + + # Set timestamp_type conversion function, used when generating queries and processing hits + rule['timestamp_type'] = rule['timestamp_type'].strip().lower() + if rule['timestamp_type'] == 'iso': + rule['ts_to_dt'] = ts_to_dt + rule['dt_to_ts'] = dt_to_ts + elif rule['timestamp_type'] == 'unix': + rule['ts_to_dt'] = unix_to_dt + rule['dt_to_ts'] = dt_to_unix + elif rule['timestamp_type'] == 'unix_ms': + rule['ts_to_dt'] = unixms_to_dt + rule['dt_to_ts'] = dt_to_unixms + elif rule['timestamp_type'] == 'custom': + def _ts_to_dt_with_format(ts): + return ts_to_dt_with_format(ts, ts_format=rule['timestamp_format']) + + def _dt_to_ts_with_format(dt): + ts = dt_to_ts_with_format(dt, ts_format=rule['timestamp_format']) + if 'timestamp_format_expr' in rule: + # eval expression passing 'ts' and 'dt' + return eval(rule['timestamp_format_expr'], {'ts': ts, 'dt': dt}) + else: + return ts + + rule['ts_to_dt'] = _ts_to_dt_with_format + rule['dt_to_ts'] = _dt_to_ts_with_format + else: + raise EAException('timestamp_type must be one of iso, unix, or unix_ms') + + # Add support for client ssl certificate auth + if 'verify_certs' in conf: + rule.setdefault('verify_certs', conf.get('verify_certs')) + rule.setdefault('ca_certs', conf.get('ca_certs')) + rule.setdefault('client_cert', conf.get('client_cert')) + rule.setdefault('client_key', conf.get('client_key')) + + # Set HipChat options from global config + rule.setdefault('hipchat_msg_color', 'red') + rule.setdefault('hipchat_domain', 'api.hipchat.com') + rule.setdefault('hipchat_notify', True) + rule.setdefault('hipchat_from', '') + rule.setdefault('hipchat_ignore_ssl_errors', False) + + # Make sure we have required options + if self.required_locals - frozenset(rule.keys()): + raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(rule.keys())))) + + if 'include' in rule and type(rule['include']) != list: + raise EAException('include option must be a list') + + if isinstance(rule.get('query_key'), list): + rule['compound_query_key'] = rule['query_key'] + rule['query_key'] = ','.join(rule['query_key']) + + if isinstance(rule.get('aggregation_key'), list): + rule['compound_aggregation_key'] = rule['aggregation_key'] + rule['aggregation_key'] = ','.join(rule['aggregation_key']) + + if isinstance(rule.get('compare_key'), list): + rule['compound_compare_key'] = rule['compare_key'] + rule['compare_key'] = ','.join(rule['compare_key']) + elif 'compare_key' in rule: + rule['compound_compare_key'] = [rule['compare_key']] + # Add QK, CK and timestamp to include + include = rule.get('include', ['*']) + if 'query_key' in rule: + include.append(rule['query_key']) + if 'compound_query_key' in rule: + include += rule['compound_query_key'] + if 'compound_aggregation_key' in rule: + include += rule['compound_aggregation_key'] + if 'compare_key' in rule: + include.append(rule['compare_key']) + if 'compound_compare_key' in rule: + include += rule['compound_compare_key'] + if 'top_count_keys' in rule: + include += rule['top_count_keys'] + include.append(rule['timestamp_field']) + rule['include'] = list(set(include)) + + # Check that generate_kibana_url is compatible with the filters + if rule.get('generate_kibana_link'): + for es_filter in rule.get('filter'): + if es_filter: + if 'not' in es_filter: + es_filter = es_filter['not'] + if 'query' in es_filter: + es_filter = es_filter['query'] + if es_filter.keys()[0] not in ('term', 'query_string', 'range'): + raise EAException( + 'generate_kibana_link is incompatible with filters other than term, query_string and range.' + 'Consider creating a dashboard and using use_kibana_dashboard instead.') + + # Check that doc_type is provided if use_count/terms_query + if rule.get('use_count_query') or rule.get('use_terms_query'): + if 'doc_type' not in rule: + raise EAException('doc_type must be specified.') + + # Check that query_key is set if use_terms_query + if rule.get('use_terms_query'): + if 'query_key' not in rule: + raise EAException('query_key must be specified with use_terms_query') + + # Warn if use_strf_index is used with %y, %M or %D + # (%y = short year, %M = minutes, %D = full date) + if rule.get('use_strftime_index'): + for token in ['%y', '%M', '%D']: + if token in rule.get('index'): + logging.warning('Did you mean to use %s in the index? ' + 'The index will be formatted like %s' % (token, + datetime.datetime.now().strftime( + rule.get('index')))) + + if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): + raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') + + def load_modules(self, rule, args=None): + """ Loads things that could be modules. Enhancements, alerts and rule type. """ + # Set match enhancements + match_enhancements = [] + for enhancement_name in rule.get('match_enhancements', []): + if enhancement_name in dir(enhancements): + enhancement = getattr(enhancements, enhancement_name) + else: + enhancement = get_module(enhancement_name) + if not issubclass(enhancement, enhancements.BaseEnhancement): + raise EAException("Enhancement module %s not a subclass of BaseEnhancement" % enhancement_name) + match_enhancements.append(enhancement(rule)) + rule['match_enhancements'] = match_enhancements + + # Convert rule type into RuleType object + if rule['type'] in self.rules_mapping: + rule['type'] = self.rules_mapping[rule['type']] + else: + rule['type'] = get_module(rule['type']) + if not issubclass(rule['type'], ruletypes.RuleType): + raise EAException('Rule module %s is not a subclass of RuleType' % (rule['type'])) + + # Make sure we have required alert and type options + reqs = rule['type'].required_options + + if reqs - frozenset(rule.keys()): + raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(rule.keys())))) + # Instantiate rule + try: + rule['type'] = rule['type'](rule, args) + except (KeyError, EAException) as e: + raise EAException('Error initializing rule %s: %s' % (rule['name'], e)), None, sys.exc_info()[2] + # Instantiate alerts only if we're not in debug mode + # In debug mode alerts are not actually sent so don't bother instantiating them + if not args or not args.debug: + rule['alert'] = self.load_alerts(rule, alert_field=rule['alert']) + + def load_configuration(self, filename, conf, args=None): + """ Load a yaml rule file and fill in the relevant fields with objects. + + :param filename: The name of a rule configuration file. + :param conf: The global configuration dictionary, used for populating defaults. + :param args: Arguments + :return: The rule configuration, a dictionary. + """ + rule = self.load_yaml(filename) + self.load_options(rule, conf, filename, args) + self.load_modules(rule, args) + return rule + + def load_alerts(self, rule, alert_field): + def normalize_config(alert): + """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. + This function normalizes them both to the latter format. """ + if isinstance(alert, basestring): + return alert, rule + elif isinstance(alert, dict): + name, config = iter(alert.items()).next() + config_copy = copy.copy(rule) + config_copy.update(config) # warning, this (intentionally) mutates the rule dict + return name, config_copy + else: + raise EAException() + + def create_alert(alert, alert_config): + alert_class = self.alerts_mapping.get(alert) or get_module(alert) + if not issubclass(alert_class, alerts.Alerter): + raise EAException('Alert module %s is not a subclass of Alerter' % alert) + missing_options = (rule['type'].required_options | alert_class.required_options) - frozenset( + alert_config or []) + if missing_options: + raise EAException('Missing required option(s): %s' % (', '.join(missing_options))) + return alert_class(alert_config) + + try: + if type(alert_field) != list: + alert_field = [alert_field] + + alert_field = [normalize_config(x) for x in alert_field] + alert_field = sorted(alert_field, key=lambda (a, b): self.alerts_order.get(a, 1)) + # Convert all alerts into Alerter objects + alert_field = [create_alert(a, b) for a, b in alert_field] + + except (KeyError, EAException) as e: + raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)), None, sys.exc_info()[2] + + return alert_field + + @staticmethod + def adjust_deprecated_values(rule): + # From rename of simple HTTP alerter + if rule.get('type') == 'simple': + rule['type'] = 'post' + if 'simple_proxy' in rule: + rule['http_post_proxy'] = rule['simple_proxy'] + if 'simple_webhook_url' in rule: + rule['http_post_url'] = rule['simple_webhook_url'] + logging.warning( + '"simple" alerter has been renamed "post" and comptability may be removed in a future release.') + + +class FileRulesLoader(RulesLoader): + def load_all(self, conf, args): + names = [] + use_rule = args.rule + + # Load each rule configuration file + rules = [] + rule_files = self.__get_file_paths(conf, use_rule) + for rule_file in rule_files: + try: + rule = self.load_configuration(rule_file, conf, args) + # By setting "is_enabled: False" in rule file, a rule is easily disabled + if 'is_enabled' in rule and not rule['is_enabled']: + continue + if rule['name'] in names: + raise EAException('Duplicate rule named %s' % (rule['name'])) + except EAException as e: + raise EAException('Error loading file %s: %s' % (rule_file, e)) + + rules.append(rule) + names.append(rule['name']) + + return rules + + def get_hashes(self, conf, use_rule=None): + rule_files = self.__get_file_paths(conf, use_rule) + rule_mod_times = {} + for rule_file in rule_files: + rule_mod_times[rule_file] = self.__get_rule_file_hash(rule_file) + return rule_mod_times + + def get_yaml(self, filename): + try: + return yaml_loader(filename) + except yaml.scanner.ScannerError as e: + raise EAException('Could not parse file %s: %s' % (filename, e)) + + def get_import_rule(self, rule): + if os.path.isabs(rule['import']): + return rule['import'] + else: + return os.path.join(os.path.dirname(rule['rule_file']), rule['import']) + + def __get_file_paths(self, conf, use_rule=None): + # Passing a filename directly can bypass rules_folder and .yaml checks + if use_rule and os.path.isfile(use_rule): + return [use_rule] + rule_folder = conf['rules_folder'] + rule_files = [] + if conf['scan_subdirectories']: + for root, folders, files in os.walk(rule_folder): + for filename in files: + if use_rule and use_rule != filename: + continue + if self.is_yaml(filename): + rule_files.append(os.path.join(root, filename)) + else: + for filename in os.listdir(rule_folder): + fullpath = os.path.join(rule_folder, filename) + if os.path.isfile(fullpath) and self.is_yaml(filename): + rule_files.append(fullpath) + return rule_files + + def __get_rule_file_hash(self, rule_file): + rule_file_hash = '' + if os.path.exists(rule_file): + with open(rule_file) as fh: + rule_file_hash = hashlib.sha1(fh.read()).digest() + for import_rule_file in self.import_rules.get(rule_file, []): + rule_file_hash += self.__get_rule_file_hash(import_rule_file) + return rule_file_hash + + @staticmethod + def is_yaml(filename): + return filename.endswith('.yaml') or filename.endswith('.yml') From 2984c34b1e0f72e8f4d4c199029e714abb1b7dbf Mon Sep 17 00:00:00 2001 From: Ryan Fang Date: Thu, 12 Jul 2018 15:44:00 -0700 Subject: [PATCH 017/264] add tag support for aggregation summry text --- elastalert/alerts.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d73bcceb7..40d48364b 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -257,6 +257,10 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = '' + summary_prefix = None + if 'summary_prefix' in self.rule and self.rule['summary_prefix']: + summary_prefix = self.rule['summary_prefix'] + text = '{{{0}}}'.format(summary_prefix) if 'aggregation' in self.rule and 'summary_table_fields' in self.rule: summary_table_fields = self.rule['summary_table_fields'] if not isinstance(summary_table_fields, list): @@ -282,6 +286,8 @@ def get_aggregation_summary_text(self, matches): for keys, count in match_aggregation.iteritems(): text_table.add_row([key for key in keys] + [count]) text += text_table.draw() + '\n\n' + if summary_prefix: + text += '{{{0}}}'.format(summary_prefix) return unicode(text) From 499d4b790ad741d9c98a6782b6e99854fbe0cd2e Mon Sep 17 00:00:00 2001 From: Ryan Fang Date: Thu, 12 Jul 2018 16:33:37 -0700 Subject: [PATCH 018/264] get rid of unnecessary {} --- elastalert/alerts.py | 10 ++-------- 1 file changed, 2 insertions(+), 8 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 40d48364b..322963dd8 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -256,11 +256,7 @@ def get_aggregation_summary_text__maximum_width(self): return 80 def get_aggregation_summary_text(self, matches): - text = '' - summary_prefix = None - if 'summary_prefix' in self.rule and self.rule['summary_prefix']: - summary_prefix = self.rule['summary_prefix'] - text = '{{{0}}}'.format(summary_prefix) + text = self.rule.get('summary_prefix', '') if 'aggregation' in self.rule and 'summary_table_fields' in self.rule: summary_table_fields = self.rule['summary_table_fields'] if not isinstance(summary_table_fields, list): @@ -286,9 +282,7 @@ def get_aggregation_summary_text(self, matches): for keys, count in match_aggregation.iteritems(): text_table.add_row([key for key in keys] + [count]) text += text_table.draw() + '\n\n' - if summary_prefix: - text += '{{{0}}}'.format(summary_prefix) - + text += self.rule.get('summary_prefix', '') return unicode(text) def create_default_title(self, matches): From 35982eb30560ce220c35e1eefb5804fc881b38d7 Mon Sep 17 00:00:00 2001 From: Ryan Fang Date: Thu, 12 Jul 2018 18:00:31 -0700 Subject: [PATCH 019/264] avoid alone prefix --- elastalert/alerts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 322963dd8..f5f496466 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -256,9 +256,9 @@ def get_aggregation_summary_text__maximum_width(self): return 80 def get_aggregation_summary_text(self, matches): - text = self.rule.get('summary_prefix', '') + text = '' if 'aggregation' in self.rule and 'summary_table_fields' in self.rule: - summary_table_fields = self.rule['summary_table_fields'] + text = self.rule.get('summary_prefix', '') if not isinstance(summary_table_fields, list): summary_table_fields = [summary_table_fields] # Include a count aggregation so that we can see at a glance how many of each aggregation_key were encountered From cda2dae64f3ccb78b54fa7f526c951779c451482 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 10:54:29 +0100 Subject: [PATCH 020/264] Fixes from testing, refactoring of names, simplification of RulesLoader and updated documentation --- docs/source/elastalert.rst | 5 +- docs/source/index.rst | 1 + docs/source/recipes/adding_loaders.rst | 84 +++++++ elastalert/config.py | 32 ++- elastalert/elastalert.py | 2 +- elastalert/{ruleloaders.py => loaders.py} | 167 +++++++------ elastalert/test_rule.py | 73 ++---- elastalert/util.py | 14 ++ tests/alerts_test.py | 77 ++++-- tests/base_test.py | 20 +- tests/conftest.py | 18 +- tests/{config_test.py => loaders_test.py} | 291 ++++++++++++---------- 12 files changed, 471 insertions(+), 313 deletions(-) create mode 100644 docs/source/recipes/adding_loaders.rst rename elastalert/{ruleloaders.py => loaders.py} (90%) rename tests/{config_test.py => loaders_test.py} (58%) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index fd2daec8c..60c031cbf 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -131,9 +131,12 @@ The environment variable ``ES_USE_SSL`` will override this field. ``es_conn_timeout``: Optional; sets timeout for connecting to and reading from ``es_host``; defaults to ``20``. +``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes. +Defaults to ``FileRulesLoader`` if not set. + ``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all files in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload -or remove rules based on their respective config files. +or remove rules based on their respective config files. (only required when using ``FileRulesLoader``). ``scan_subdirectories``: Optional; Sets whether or not ElastAlert should recursively descend the rules directory - ``true`` or ``false``. The default is ``true`` diff --git a/docs/source/index.rst b/docs/source/index.rst index c3f02f535..4219bf13e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,6 +19,7 @@ Contents: recipes/adding_alerts recipes/writing_filters recipes/adding_enhancements + recipes/adding_loaders recipes/signing_requests Indices and Tables diff --git a/docs/source/recipes/adding_loaders.rst b/docs/source/recipes/adding_loaders.rst new file mode 100644 index 000000000..c77520551 --- /dev/null +++ b/docs/source/recipes/adding_loaders.rst @@ -0,0 +1,84 @@ +.. _loaders: + +Rules Loaders +======================== + +RulesLoaders are subclasses of ``RulesLoader``, found in ``elastalert/loaders.py``. They are used to +gather rules for a particular source. Your RulesLoader needs to implement three member functions, and +will look something like this: + +.. code-block:: python + + class AwesomeNewRulesLoader(RulesLoader): + def get_names(self, conf, use_rule=None): + ... + def get_hashes(self, conf, use_rule=None): + ... + def get_yaml(self, rule): + ... + +You can import loaders by specifying the type as ``module.file.RulesLoaderName``, where module is the name of a +python module, and file is the name of the python file containing a ``RulesLoader`` subclass named ``RulesLoaderName``. + +Example +------- + +As an example loader, let's retrieve rules from a database rather than from the local file system. First, create a +modules folder for the loader in the ElastAlert directory. + +.. code-block:: console + + $ mkdir elastalert_modules + $ cd elastalert_modules + $ touch __init__.py + +Now, in a file named ``mongo_loader.py``, add + +.. code-block:: python + + from pymongo import MongoClient + from elastalert.loaders import RulesLoader + + class MongoRulesLoader(RulesLoader): + def __init__(self, conf): + super(MongoRulesLoader, self).__init__(conf) + self.client = MongoClient(conf['mongo_url']) + self.db = client[conf['mongo_db']] + + def get_names(self, conf, use_rule=None): + if use_rule: + return [use_rule] + + rules = [] + self.cache = {} + for rule in self.db.rules.find(): + self.cache[rule.name] = rule.yaml + rules.append(rule.yaml) + + return rules + + + def get_hashes(self, conf, use_rule=None): + if use_rule: + return [use_rule] + + hashes = {} + self.cache = {} + for rule in self.db.rules.find(): + self.cache[rule.name] = rule.yaml + hashes[rule.name] = rule.hash + + return hashes + + def get_yaml(self, rule): + if rule in self.cache: + return self.cache[rule] + + self.cache[rule] = self.db.rules.find_one({'name': rule}).yaml + return self.cache[rule] + +Finally, you need to specify in your ElastAlert configuration file that MongoRulesLoader should be used instead of the +default FileRulesLoader, so in your ``elastalert.conf`` file:: + + rules_loader: "elastalert_modules.mongo_loader.MongoRulesLoader" + diff --git a/elastalert/config.py b/elastalert/config.py index 0e21fd42b..518915bf3 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -1,15 +1,15 @@ # -*- coding: utf-8 -*- import datetime -import sys -import ruleloaders +import loaders from envparse import Env from staticconf.loader import yaml_loader from util import EAException +from util import get_module # Required global (config.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) +required_globals = frozenset(['run_every', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) # Settings that can be derived from ENV variables env_settings = {'ES_USE_SSL': 'use_ssl', @@ -23,28 +23,16 @@ # Used to map the names of rule loaders to their classes loader_mapping = { - 'file': ruleloaders.FileRulesLoader + 'file': loaders.FileRulesLoader, } -def get_module(module_name): - """ Loads a module and returns a specific object. - module_name should 'module.file.object'. - Returns object or raises EAException on error. """ - try: - module_path, module_class = module_name.rsplit('.', 1) - base_module = __import__(module_path, globals(), locals(), [module_class]) - module = getattr(base_module, module_class) - except (ImportError, AttributeError, ValueError) as e: - raise EAException("Could not import module %s: %s" % (module_name, e)), None, sys.exc_info()[2] - return module - - -def load_conf(args): +def load_conf(args, overrides=None): """ Creates a conf dictionary for ElastAlerter. Loads the global config file and then each rule found in rules_folder. :param args: The parsed arguments to ElastAlert + :param overrides: Dictionary of conf values to override :return: The global configuration, a dictionary. """ filename = args.config @@ -55,6 +43,9 @@ def load_conf(args): if val is not None: conf[conf_var] = val + for key, value in (overrides if overrides is not None else []): + conf[key] = value + # Make sure we have all required globals if required_globals - frozenset(conf.keys()): raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) @@ -84,5 +75,10 @@ def load_conf(args): rules_loader_class = loader_mapping.get(conf['rules_loader']) or get_module(conf['rules_loader']) rules_loader = rules_loader_class(conf) conf['rules_loader'] = rules_loader + # Make sure we have all the required globals for the loader + # Make sure we have all required globals + if rules_loader.required_globals - frozenset(conf.keys()): + raise EAException( + '%s must contain %s' % (filename, ', '.join(rules_loader.required_globals - frozenset(conf.keys())))) return conf diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index abcc027bf..7a1503020 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -127,7 +127,7 @@ def __init__(self, args): self.conf = load_conf(self.args) self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] - self.rules = self.conf['rules_loader'].load_all(self.conf, self.args) + self.rules = self.conf['rules_loader'].load(self.conf, self.args) self.writeback_index = self.conf['writeback_index'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] diff --git a/elastalert/ruleloaders.py b/elastalert/loaders.py similarity index 90% rename from elastalert/ruleloaders.py rename to elastalert/loaders.py index fccab7605..c4037e997 100644 --- a/elastalert/ruleloaders.py +++ b/elastalert/loaders.py @@ -14,7 +14,7 @@ import yaml.scanner from opsgenie import OpsGenieAlerter from staticconf.loader import yaml_loader -from config import get_module +from util import get_module from util import dt_to_ts from util import dt_to_ts_with_format from util import dt_to_unix @@ -30,6 +30,9 @@ class RulesLoader(object): # import rule dependency import_rules = {} + # Required global (config.yaml) configuration options for the loader + required_globals = frozenset([]) + # Required local (rule.yaml) configuration options required_locals = frozenset(['alert', 'type', 'name', 'index']) @@ -83,48 +86,98 @@ class RulesLoader(object): def __init__(self, conf): # schema for rule yaml - self.rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) + self.rule_schema = jsonschema.Draft4Validator( + yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) self.base_config = copy.deepcopy(conf) - def load_all(self, conf, args): + def load(self, conf, args=None): """ - Load all the rules and return them. - :param conf: Configuration dict - :param args: Arguments dict + Discover and load all the rules as defined in the conf and args. + :param dict conf: Configuration dict + :param dict args: Arguments dict :return: List of rules + :rtype: list + """ + names = [] + use_rule = None if args is None else args.rule + + # Load each rule configuration file + rules = [] + rule_files = self.get_names(conf, use_rule) + for rule_file in rule_files: + try: + rule = self.load_configuration(rule_file, conf, args) + # By setting "is_enabled: False" in rule file, a rule is easily disabled + if 'is_enabled' in rule and not rule['is_enabled']: + continue + if rule['name'] in names: + raise EAException('Duplicate rule named %s' % (rule['name'])) + except EAException as e: + raise EAException('Error loading file %s: %s' % (rule_file, e)) + + rules.append(rule) + names.append(rule['name']) + + return rules + + def get_names(self, conf, use_rule=None): + """ + Return a list of rule names that can be passed to `get_yaml` to retrieve. + :param dict conf: Configuration dict + :param str use_rule: Limit to only specified rule + :return: A list of rule names + :rtype: list """ raise NotImplementedError() def get_hashes(self, conf, use_rule=None): """ - Get hashes of the rules. - :param conf: Configuration dict - :param use_rule: Limit to only specified rule + Discover and get the hashes of all the rules as defined in the conf. + :param dict conf: Configuration + :param str use_rule: Limit to only specified rule :return: Dict of rule name to hash + :rtype: dict """ raise NotImplementedError() def get_yaml(self, filename): """ Get and parse the yaml of the specified rule. - :param filename: Rule to get the yaml + :param str filename: Rule to get the yaml :return: Rule YAML dict + :rtype: dict """ raise NotImplementedError() def get_import_rule(self, rule): """ - :param rule: Rule dict + Retrieve the name of the rule to import. + :param dict rule: Rule dict :return: rule name that will all `get_yaml` to retrieve the yaml of the rule + :rtype: str """ - raise NotImplementedError() + return rule['import'] + + def load_configuration(self, filename, conf, args=None): + """ Load a yaml rule file and fill in the relevant fields with objects. + + :param str filename: The name of a rule configuration file. + :param dict conf: The global configuration dictionary, used for populating defaults. + :param dict args: Arguments + :return: The rule configuration, a dictionary. + """ + rule = self.load_yaml(filename) + self.load_options(rule, conf, filename, args) + self.load_modules(rule, args) + return rule def load_yaml(self, filename): """ Load the rule including all dependency rules. - :param filename: Rule to load + :param str filename: Rule to load :return: Loaded rule dict + :rtype: dict """ rule = { 'rule_file': filename, @@ -364,19 +417,6 @@ def load_modules(self, rule, args=None): if not args or not args.debug: rule['alert'] = self.load_alerts(rule, alert_field=rule['alert']) - def load_configuration(self, filename, conf, args=None): - """ Load a yaml rule file and fill in the relevant fields with objects. - - :param filename: The name of a rule configuration file. - :param conf: The global configuration dictionary, used for populating defaults. - :param args: Arguments - :return: The rule configuration, a dictionary. - """ - rule = self.load_yaml(filename) - self.load_options(rule, conf, filename, args) - self.load_modules(rule, args) - return rule - def load_alerts(self, rule, alert_field): def normalize_config(alert): """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. @@ -429,55 +469,17 @@ def adjust_deprecated_values(rule): class FileRulesLoader(RulesLoader): - def load_all(self, conf, args): - names = [] - use_rule = args.rule - - # Load each rule configuration file - rules = [] - rule_files = self.__get_file_paths(conf, use_rule) - for rule_file in rule_files: - try: - rule = self.load_configuration(rule_file, conf, args) - # By setting "is_enabled: False" in rule file, a rule is easily disabled - if 'is_enabled' in rule and not rule['is_enabled']: - continue - if rule['name'] in names: - raise EAException('Duplicate rule named %s' % (rule['name'])) - except EAException as e: - raise EAException('Error loading file %s: %s' % (rule_file, e)) - - rules.append(rule) - names.append(rule['name']) - - return rules - - def get_hashes(self, conf, use_rule=None): - rule_files = self.__get_file_paths(conf, use_rule) - rule_mod_times = {} - for rule_file in rule_files: - rule_mod_times[rule_file] = self.__get_rule_file_hash(rule_file) - return rule_mod_times - - def get_yaml(self, filename): - try: - return yaml_loader(filename) - except yaml.scanner.ScannerError as e: - raise EAException('Could not parse file %s: %s' % (filename, e)) - def get_import_rule(self, rule): - if os.path.isabs(rule['import']): - return rule['import'] - else: - return os.path.join(os.path.dirname(rule['rule_file']), rule['import']) + # Required global (config.yaml) configuration options for the loader + required_globals = frozenset(['rules_folder']) - def __get_file_paths(self, conf, use_rule=None): + def get_names(self, conf, use_rule=None): # Passing a filename directly can bypass rules_folder and .yaml checks if use_rule and os.path.isfile(use_rule): return [use_rule] rule_folder = conf['rules_folder'] rule_files = [] - if conf['scan_subdirectories']: + if 'scan_subdirectories' in conf and conf['scan_subdirectories']: for root, folders, files in os.walk(rule_folder): for filename in files: if use_rule and use_rule != filename: @@ -491,13 +493,38 @@ def __get_file_paths(self, conf, use_rule=None): rule_files.append(fullpath) return rule_files - def __get_rule_file_hash(self, rule_file): + def get_hashes(self, conf, use_rule=None): + rule_files = self.get_names(conf, use_rule) + rule_mod_times = {} + for rule_file in rule_files: + rule_mod_times[rule_file] = self.get_rule_file_hash(rule_file) + return rule_mod_times + + def get_yaml(self, filename): + try: + return yaml_loader(filename) + except yaml.scanner.ScannerError as e: + raise EAException('Could not parse file %s: %s' % (filename, e)) + + def get_import_rule(self, rule): + """ + Allow for relative paths to the import rule. + :param dict rule: + :return: Path the import rule + :rtype: str + """ + if os.path.isabs(rule['import']): + return rule['import'] + else: + return os.path.join(os.path.dirname(rule['rule_file']), rule['import']) + + def get_rule_file_hash(self, rule_file): rule_file_hash = '' if os.path.exists(rule_file): with open(rule_file) as fh: rule_file_hash = hashlib.sha1(fh.read()).digest() for import_rule_file in self.import_rules.get(rule_file, []): - rule_file_hash += self.__get_rule_file_hash(import_rule_file) + rule_file_hash += self.get_rule_file_hash(import_rule_file) return rule_file_hash @staticmethod diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 654146aea..631b97d54 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -7,7 +7,6 @@ import datetime import json import logging -import os import random import re import string @@ -15,12 +14,8 @@ import argparse import mock -import yaml -import elastalert.config -from elastalert.config import load_modules -from elastalert.config import load_options -from elastalert.config import load_rule_yaml +from elastalert.config import load_conf from elastalert.elastalert import ElastAlerter from elastalert.util import elasticsearch_client from elastalert.util import lookup_es_key @@ -211,7 +206,7 @@ def run_elastalert(self, rule, conf, args): # It is needed to prevent unnecessary initialization of unused alerters load_modules_args = argparse.Namespace() load_modules_args.debug = not args.alert - load_modules(rule, load_modules_args) + conf['rules_loader'].load_modules(rule, load_modules_args) conf['rules'] = [rule] # If using mock data, make sure it's sorted and find appropriate time range @@ -281,50 +276,6 @@ def get_id(): if errors and args.stop_error: exit(1) - def load_conf(self, rules, args): - """ Loads a default conf dictionary (from global config file, if provided, or hard-coded mocked data), - for initializing rules. Also initializes rules. - - :return: the default rule configuration, a dictionary """ - if args.config is not None: - with open(args.config) as fh: - conf = yaml.load(fh) - else: - if os.path.isfile('config.yaml'): - with open('config.yaml') as fh: - conf = yaml.load(fh) - else: - conf = {} - - # Need to convert these parameters to datetime objects - for key in ['buffer_time', 'run_every', 'alert_time_limit', 'old_query_limit']: - if key in conf: - conf[key] = datetime.timedelta(**conf[key]) - - # Mock configuration. This specifies the base values for attributes, unless supplied otherwise. - conf_default = { - 'rules_folder': 'rules', - 'es_host': 'localhost', - 'es_port': 14900, - 'writeback_index': 'wb', - 'max_query_size': 10000, - 'alert_time_limit': datetime.timedelta(hours=24), - 'old_query_limit': datetime.timedelta(weeks=1), - 'run_every': datetime.timedelta(minutes=5), - 'disable_rules_on_error': False, - 'buffer_time': datetime.timedelta(minutes=45), - 'scroll_keepalive': '30s' - } - - for key in conf_default: - if key not in conf: - conf[key] = conf_default[key] - elastalert.config.base_config = copy.deepcopy(conf) - load_options(rules, conf, args.file) - print("Successfully loaded %s\n" % (rules['name'])) - - return conf - def run_rule_test(self): """ Uses args to run the various components of MockElastAlerter such as loading the file, saving data, loading data, and running. @@ -357,9 +308,25 @@ def run_rule_test(self): parser.add_argument('--config', action='store', dest='config', help='Global config file.') args = parser.parse_args() - rule_yaml = load_rule_yaml(args.file) + # rule_yaml = load_rule_yaml(args.file) - conf = self.load_conf(rule_yaml, args) + # conf = self.load_conf(rule_yaml, args) + overrides = { + 'rules_folder': 'rules', + 'es_host': 'localhost', + 'es_port': 14900, + 'writeback_index': 'wb', + 'max_query_size': 10000, + 'alert_time_limit': datetime.timedelta(hours=24), + 'old_query_limit': datetime.timedelta(weeks=1), + 'run_every': datetime.timedelta(minutes=5), + 'disable_rules_on_error': False, + 'buffer_time': datetime.timedelta(minutes=45), + 'scroll_keepalive': '30s' + } + conf = load_conf(args, overrides) + rule_yaml = conf['rules_loader'].get_yaml(args.file) + conf['rules_loader'].load_options(rule_yaml, conf, args.file) if args.json: with open(args.json, 'r') as data_file: diff --git a/elastalert/util.py b/elastalert/util.py index 1751cb6a9..a62475fc5 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -3,6 +3,7 @@ import datetime import logging import os +import sys import dateutil.parser import dateutil.tz @@ -15,6 +16,19 @@ elastalert_logger = logging.getLogger('elastalert') +def get_module(module_name): + """ Loads a module and returns a specific object. + module_name should 'module.file.object'. + Returns object or raises EAException on error. """ + try: + module_path, module_class = module_name.rsplit('.', 1) + base_module = __import__(module_path, globals(), locals(), [module_class]) + module = getattr(base_module, module_class) + except (ImportError, AttributeError, ValueError) as e: + raise EAException("Could not import module %s: %s" % (module_name, e)), None, sys.exc_info()[2] + return module + + def new_get_event_ts(ts_field): """ Constructs a lambda that may be called to extract the timestamp field from a given event. diff --git a/tests/alerts_test.py b/tests/alerts_test.py index b5f32f4a6..73c27f6dc 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -21,7 +21,7 @@ from elastalert.alerts import PagerDutyAlerter from elastalert.alerts import SlackAlerter from elastalert.alerts import StrideAlerter -from elastalert.config import load_modules +from elastalert.loaders import FileRulesLoader from elastalert.opsgenie import OpsGenieAlerter from elastalert.util import ts_add from elastalert.util import ts_now @@ -866,7 +866,8 @@ def test_ms_teams(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = MsTeamsAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -902,7 +903,8 @@ def test_ms_teams_uses_color_and_fixed_width_text(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = MsTeamsAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -938,7 +940,8 @@ def test_slack_uses_custom_title(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -979,7 +982,8 @@ def test_slack_uses_rule_name_when_custom_title_is_not_provided(): 'slack_webhook_url': ['http://please.dontgohere.slack'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1021,7 +1025,8 @@ def test_slack_uses_custom_slack_channel(): 'slack_channel_override': '#test-alert', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1064,7 +1069,8 @@ def test_http_alerter_with_payload(): 'http_post_static_payload': {'name': 'somestaticname'}, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1095,7 +1101,8 @@ def test_http_alerter_with_payload_all_values(): 'http_post_all_values': True, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1126,7 +1133,8 @@ def test_http_alerter_without_payload(): 'http_post_static_payload': {'name': 'somestaticname'}, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1156,7 +1164,8 @@ def test_pagerduty_alerter(): 'pagerduty_client_name': 'ponies inc.', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1187,7 +1196,8 @@ def test_pagerduty_alerter_custom_incident_key(): 'pagerduty_incident_key': 'custom key', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1219,7 +1229,8 @@ def test_pagerduty_alerter_custom_incident_key_with_args(): 'pagerduty_incident_key_args': ['somefield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1252,7 +1263,8 @@ def test_pagerduty_alerter_custom_alert_subject(): 'pagerduty_incident_key_args': ['somefield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1286,7 +1298,8 @@ def test_pagerduty_alerter_custom_alert_subject_with_args(): 'pagerduty_incident_key_args': ['someotherfield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1322,7 +1335,8 @@ def test_pagerduty_alerter_custom_alert_subject_with_args_specifying_trigger(): 'pagerduty_incident_key_args': ['someotherfield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1447,7 +1461,8 @@ def test_stride_plain_text(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1492,7 +1507,8 @@ def test_stride_underline_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1537,7 +1553,8 @@ def test_stride_bold_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1582,7 +1599,8 @@ def test_stride_strong_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1627,7 +1645,8 @@ def test_stride_hyperlink(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1672,7 +1691,8 @@ def test_stride_html(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1724,7 +1744,8 @@ def test_hipchat_body_size_limit_text(): 'message': 'message', }, } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HipChatAlerter(rule) match = { '@timestamp': '2018-01-01T00:00:00', @@ -1751,7 +1772,8 @@ def test_hipchat_body_size_limit_html(): 'message': 'message', }, } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HipChatAlerter(rule) match = { '@timestamp': '2018-01-01T00:00:00', @@ -1791,7 +1813,8 @@ def test_alerta_no_auth(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) @@ -1844,7 +1867,8 @@ def test_alerta_auth(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) @@ -1886,7 +1910,8 @@ def test_alerta_new_style(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) diff --git a/tests/base_test.py b/tests/base_test.py index 52d7ae1f3..cc72b17e3 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -925,8 +925,8 @@ def test_rule_changes(ea): 'rules/rule3.yaml': 'XXX', 'rules/rule2.yaml': '!@#$'} - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml'}, {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml'}] mock_hashes.return_value = new_hashes @@ -946,8 +946,8 @@ def test_rule_changes(ea): # A new rule with a conflicting name wont load new_hashes = copy.copy(new_hashes) new_hashes.update({'rules/rule4.yaml': 'asdf'}) - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: with mock.patch.object(ea, 'send_notification_email') as mock_send: mock_load.return_value = {'filter': [], 'name': 'rule3', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes @@ -959,8 +959,8 @@ def test_rule_changes(ea): # A new rule with is_enabled=False wont load new_hashes = copy.copy(new_hashes) new_hashes.update({'rules/rule4.yaml': 'asdf'}) - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes ea.load_rule_changes() @@ -970,8 +970,8 @@ def test_rule_changes(ea): # An old rule which didn't load gets reloaded new_hashes = copy.copy(new_hashes) new_hashes['rules/rule4.yaml'] = 'qwerty' - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes ea.load_rule_changes() @@ -1191,8 +1191,8 @@ def test_uncaught_exceptions(ea): # Changing the file should re-enable it ea.rule_hashes = {'blah.yaml': 'abc'} new_hashes = {'blah.yaml': 'def'} - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.side_effect = [ea.disabled_rules[0]] mock_hashes.return_value = new_hashes ea.load_rule_changes() diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..157a8e4ba 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -51,6 +51,14 @@ def __init__(self, host='es', port=14900): self.indices = mock_es_indices_client() +class mock_rule_loader(object): + def __init__(self, conf): + self.base_config = conf + self.load = mock.Mock() + self.get_hashes = mock.Mock() + self.load_configuration = mock.Mock() + + class mock_ruletype(object): def __init__(self): self.add_data = mock.Mock() @@ -100,11 +108,13 @@ def ea(): 'old_query_limit': datetime.timedelta(weeks=1), 'disable_rules_on_error': False, 'scroll_keepalive': '30s'} + conf['rules_loader'] = mock_rule_loader(conf) elastalert.elastalert.elasticsearch_client = mock_es_client - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: - load_conf.return_value = conf - ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + load_conf.return_value = conf + conf['rules_loader'].load.return_value = rules + conf['rules_loader'].get_hashes.return_value = {} + ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) ea.rules[0]['type'] = mock_ruletype() ea.rules[0]['alert'] = [mock_alert()] ea.writeback_es = mock_es_client() diff --git a/tests/config_test.py b/tests/loaders_test.py similarity index 58% rename from tests/config_test.py rename to tests/loaders_test.py index f444f0e25..2d317d455 100644 --- a/tests/config_test.py +++ b/tests/loaders_test.py @@ -8,13 +8,9 @@ import elastalert.alerts import elastalert.ruletypes -from elastalert.config import get_file_paths -from elastalert.config import load_configuration -from elastalert.config import load_modules -from elastalert.config import load_options -from elastalert.config import load_rules +from elastalert.config import load_conf +from elastalert.loaders import FileRulesLoader from elastalert.util import EAException -from elastalert.config import import_rules test_config = {'rules_folder': 'test_folder', 'run_every': {'minutes': 10}, @@ -48,15 +44,16 @@ def test_import_rules(): + rules_loader = FileRulesLoader(test_config) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['type'] = 'testing.test.RuleType' - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'load_yaml') as mock_open: mock_open.return_value = test_rule_copy # Test that type is imported with mock.patch('__builtin__.__import__') as mock_import: mock_import.return_value = elastalert.ruletypes - load_configuration('test_config', test_config) + rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing.test' assert mock_import.call_args_list[0][0][3] == ['RuleType'] @@ -66,12 +63,13 @@ def test_import_rules(): test_rule_copy['alert'] = 'testing2.test2.Alerter' with mock.patch('__builtin__.__import__') as mock_import: mock_import.return_value = elastalert.alerts - load_configuration('test_config', test_config) + rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing2.test2' assert mock_import.call_args_list[0][0][3] == ['Alerter'] def test_import_import(): + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -82,9 +80,9 @@ def test_import_import(): 'email': 'ignored@email', # overwritten by the email in import_rule } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert mock_open.call_args_list[0][0] == ('blah.yaml',) assert mock_open.call_args_list[1][0] == ('importme.ymlt',) assert len(mock_open.call_args_list) == 2 @@ -94,10 +92,11 @@ def test_import_import(): assert rules['filter'] == import_rule['filter'] # check global import_rule dependency - assert import_rules == {'blah.yaml': ['importme.ymlt']} + assert rules_loader.import_rules == {'blah.yaml': ['importme.ymlt']} def test_import_absolute_import(): + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -108,9 +107,9 @@ def test_import_absolute_import(): 'email': 'ignored@email', # overwritten by the email in import_rule } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert mock_open.call_args_list[0][0] == ('blah.yaml',) assert mock_open.call_args_list[1][0] == ('/importme.ymlt',) assert len(mock_open.call_args_list) == 2 @@ -123,6 +122,7 @@ def test_import_absolute_import(): def test_import_filter(): # Check that if a filter is specified the rules are merged: + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -133,13 +133,14 @@ def test_import_filter(): 'filter': [{'term': {'ratchet': 'clank'}}], } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert rules['filter'] == [{'term': {'ratchet': 'clank'}}, {'term': {'key': 'value'}}] def test_load_inline_alert_rule(): + rules_loader = FileRulesLoader(test_config) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['alert'] = [ { @@ -154,34 +155,75 @@ def test_load_inline_alert_rule(): } ] test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [test_config_copy, test_rule_copy] - load_modules(test_rule_copy) + rules_loader.load_modules(test_rule_copy) assert isinstance(test_rule_copy['alert'][0], elastalert.alerts.EmailAlerter) assert isinstance(test_rule_copy['alert'][1], elastalert.alerts.EmailAlerter) assert 'foo@bar.baz' in test_rule_copy['alert'][0].rule['email'] assert 'baz@foo.bar' in test_rule_copy['alert'][1].rule['email'] +def test_file_rules_loader_get_names_recursive(): + conf = {'scan_subdirectories': True, 'rules_folder': 'root'} + rules_loader = FileRulesLoader(conf) + walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)), + ('root/folder_a', (), ('a.yaml', 'ab.yaml')), + ('root/folder_b', (), ('b.yaml',))) + with mock.patch('os.walk') as mock_walk: + mock_walk.return_value = walk_paths + paths = rules_loader.get_names(conf) + + paths = [p.replace(os.path.sep, '/') for p in paths] + + assert 'root/rule.yaml' in paths + assert 'root/folder_a/a.yaml' in paths + assert 'root/folder_a/ab.yaml' in paths + assert 'root/folder_b/b.yaml' in paths + assert len(paths) == 4 + + +def test_file_rules_loader_get_names(): + # Check for no subdirectory + conf = {'scan_subdirectories': False, 'rules_folder': 'root'} + rules_loader = FileRulesLoader(conf) + files = ['badfile', 'a.yaml', 'b.yaml'] + + with mock.patch('os.listdir') as mock_list: + with mock.patch('os.path.isfile') as mock_path: + mock_path.return_value = True + mock_list.return_value = files + paths = rules_loader.get_names(conf) + + paths = [p.replace(os.path.sep, '/') for p in paths] + + assert 'root/a.yaml' in paths + assert 'root/b.yaml' in paths + assert len(paths) == 2 + + def test_load_rules(): test_rule_copy = copy.deepcopy(test_rule) test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) - assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) - assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter) - assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta) - assert isinstance(rules['run_every'], datetime.timedelta) - for included_key in ['comparekey', 'testkey', '@timestamp']: - assert included_key in rules['rules'][0]['include'] + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) + assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter) + assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta) + assert isinstance(rules['run_every'], datetime.timedelta) + for included_key in ['comparekey', 'testkey', '@timestamp']: + assert included_key in rules['rules'][0]['include'] - # Assert include doesn't contain duplicates - assert rules['rules'][0]['include'].count('@timestamp') == 1 - assert rules['rules'][0]['include'].count('comparekey') == 1 + # Assert include doesn't contain duplicates + assert rules['rules'][0]['include'].count('@timestamp') == 1 + assert rules['rules'][0]['include'].count('comparekey') == 1 def test_load_default_host_port(): @@ -189,16 +231,19 @@ def test_load_default_host_port(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - # Assert include doesn't contain duplicates - assert rules['es_port'] == 12345 - assert rules['es_host'] == 'elasticsearch.test' + # Assert include doesn't contain duplicates + assert rules['es_port'] == 12345 + assert rules['es_host'] == 'elasticsearch.test' def test_load_ssl_env_false(): @@ -206,15 +251,18 @@ def test_load_ssl_env_false(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_USE_SSL': 'false'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_USE_SSL': 'false'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['use_ssl'] is False + assert rules['use_ssl'] is False def test_load_ssl_env_true(): @@ -222,15 +270,18 @@ def test_load_ssl_env_true(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_USE_SSL': 'true'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_USE_SSL': 'true'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['use_ssl'] is True + assert rules['use_ssl'] is True def test_load_url_prefix_env(): @@ -238,36 +289,68 @@ def test_load_url_prefix_env(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_URL_PREFIX': 'es/'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_URL_PREFIX': 'es/'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['es_url_prefix'] == 'es/' + assert rules['es_url_prefix'] == 'es/' def test_load_disabled_rules(): test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['is_enabled'] = False test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + # The rule is not loaded for it has "is_enabled=False" + assert len(rules['rules']) == 0 + + +def test_raises_on_missing_config(): + optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') + test_rule_copy = copy.deepcopy(test_rule) + for key in test_rule_copy.keys(): + test_rule_copy = copy.deepcopy(test_rule) + test_config_copy = copy.deepcopy(test_config) + test_rule_copy.pop(key) + + # Non required keys + if key in optional_keys: + continue - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) - # The rule is not loaded for it has "is_enabled=False" - assert len(rules['rules']) == 0 + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + with pytest.raises(EAException, message='key %s should be required' % key): + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + print(rules) def test_compound_query_key(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy.pop('use_count_query') test_rule_copy['query_key'] = ['field1', 'field2'] - load_options(test_rule_copy, test_config, 'filename.yaml') + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') assert 'field1' in test_rule_copy['include'] assert 'field2' in test_rule_copy['include'] assert test_rule_copy['query_key'] == 'field1,field2' @@ -275,33 +358,14 @@ def test_compound_query_key(): def test_name_inference(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy.pop('name') - load_options(test_rule_copy, test_config, 'msmerc woz ere.yaml') + rules_loader.load_options(test_rule_copy, test_config, 'msmerc woz ere.yaml') assert test_rule_copy['name'] == 'msmerc woz ere' -def test_raises_on_missing_config(): - optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') - test_rule_copy = copy.deepcopy(test_rule) - for key in test_rule_copy.keys(): - test_rule_copy = copy.deepcopy(test_rule) - test_config_copy = copy.deepcopy(test_config) - test_rule_copy.pop(key) - - # Non required keys - if key in optional_keys: - continue - - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - with pytest.raises(EAException, message='key %s should be required' % key): - rule = load_rules(test_args) - print(rule) - - def test_raises_on_bad_generate_kibana_filters(): test_rule['generate_kibana_link'] = True bad_filters = [[{'not': {'terms': {'blah': 'blah'}}}], @@ -318,48 +382,15 @@ def test_raises_on_bad_generate_kibana_filters(): # Test that all the good filters work, but fail with a bad filter added for good in good_filters: + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['filter'] = good - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.return_value = test_rule_copy - load_configuration('blah', test_config) + rules_loader.load_configuration('blah', test_config) for bad in bad_filters: test_rule_copy['filter'] = good + bad with pytest.raises(EAException): - load_configuration('blah', test_config) - - -def test_get_file_paths_recursive(): - conf = {'scan_subdirectories': True, 'rules_folder': 'root'} - walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)), - ('root/folder_a', (), ('a.yaml', 'ab.yaml')), - ('root/folder_b', (), ('b.yaml',))) - with mock.patch('os.walk') as mock_walk: - mock_walk.return_value = walk_paths - paths = get_file_paths(conf) - - paths = [p.replace(os.path.sep, '/') for p in paths] - - assert 'root/rule.yaml' in paths - assert 'root/folder_a/a.yaml' in paths - assert 'root/folder_a/ab.yaml' in paths - assert 'root/folder_b/b.yaml' in paths - assert len(paths) == 4 - - -def test_get_file_paths(): - # Check for no subdirectory - conf = {'scan_subdirectories': False, 'rules_folder': 'root'} - files = ['badfile', 'a.yaml', 'b.yaml'] - - with mock.patch('os.listdir') as mock_list: - with mock.patch('os.path.isfile') as mock_path: - mock_path.return_value = True - mock_list.return_value = files - paths = get_file_paths(conf) - - paths = [p.replace(os.path.sep, '/') for p in paths] - - assert 'root/a.yaml' in paths - assert 'root/b.yaml' in paths - assert len(paths) == 2 + rules_loader.load_configuration('blah', test_config) From 19f3d0dfb89244a7e0689519d9ac1c91f35ed5fd Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 11:25:08 +0100 Subject: [PATCH 021/264] Fixing tests after merging with v0.1.33 --- tests/alerts_test.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 9a929fd90..a64195b78 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1205,7 +1205,8 @@ def test_pagerduty_alerter_v2(): 'pagerduty_v2_payload_source': 'mysql.host.name', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', From e8d4422f4d346e8bb32e761977f52f2f6c47185c Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 11:41:19 +0100 Subject: [PATCH 022/264] converted writeback_alias to optional conf value with default and applied strftime directly to suffix --- elastalert/config.py | 4 ++-- elastalert/elastalert.py | 7 +------ 2 files changed, 3 insertions(+), 8 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index 8fb03f71a..3a8632ff6 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -29,8 +29,7 @@ rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) # Required global (config.yaml) and local (rule.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', - 'writeback_alias', 'buffer_time']) +required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) required_locals = frozenset(['alert', 'type', 'name', 'index']) # Settings that can be derived from ENV variables @@ -448,6 +447,7 @@ def load_rules(args): if required_globals - frozenset(conf.keys()): raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) + conf.setdefault('writeback_alias', 'elastalert_alerts') conf.setdefault('max_query_size', 10000) conf.setdefault('scroll_keepalive', '30s') conf.setdefault('disable_rules_on_error', True) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 63d5dbccd..78d7525bc 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -213,12 +213,7 @@ def get_writeback_index(self, doc_type, rule=None): elif doc_type == 'elastalert_error': writeback_index += '_error' else: - suffix = rule['writeback_suffix'] - if '%' in rule['writeback_suffix']: - format_start = suffix.find('%') - format_end = suffix.rfind('%') + 2 - ts = datetime.datetime.utcnow().strftime(suffix[format_start:format_end]) - suffix = suffix[:format_start] + ts + suffix[format_end:] + suffix = datetime.datetime.utcnow().strftime(rule['writeback_suffix']) writeback_index += '_' + suffix return writeback_index From e16fcbf25014eb3c38f5acf28d76b595ecf5b8db Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 13:49:58 +0100 Subject: [PATCH 023/264] fixing bug in get hashes in elastalert __init__ --- elastalert/elastalert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 446416b3b..551f79c9a 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -144,7 +144,7 @@ def __init__(self, args): self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} - self.rule_hashes = self.conf['rules_loader'].get_hashes(self.conf, self.args) + self.rule_hashes = self.conf['rules_loader'].get_hashes(self.conf, self.args.rule) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) From 2f59d19b9e91d575afd7f74f6e2ca014ee21f14e Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Fri, 13 Jul 2018 14:58:30 +0100 Subject: [PATCH 024/264] fixes to testing a rule, EA now stores rules_loader as property --- elastalert/config.py | 9 +++++---- elastalert/elastalert.py | 14 +++++++------- elastalert/test_rule.py | 32 +++++++++++++++----------------- 3 files changed, 27 insertions(+), 28 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index 518915bf3..6d803fc60 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -27,12 +27,12 @@ } -def load_conf(args, overrides=None): +def load_conf(args, defaults=None): """ Creates a conf dictionary for ElastAlerter. Loads the global config file and then each rule found in rules_folder. :param args: The parsed arguments to ElastAlert - :param overrides: Dictionary of conf values to override + :param defaults: Dictionary of conf values to override :return: The global configuration, a dictionary. """ filename = args.config @@ -43,8 +43,9 @@ def load_conf(args, overrides=None): if val is not None: conf[conf_var] = val - for key, value in (overrides if overrides is not None else []): - conf[key] = value + for key, value in (defaults.iteritems() if defaults is not None else []): + if key not in conf: + conf[key] = value # Make sure we have all required globals if required_globals - frozenset(conf.keys()): diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 551f79c9a..2ae23724a 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -125,9 +125,10 @@ def __init__(self, args): tracer.addHandler(logging.FileHandler(self.args.es_debug_trace)) self.conf = load_conf(self.args) + self.rules_loader = self.conf['rules_loader'] self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] - self.rules = self.conf['rules_loader'].load(self.conf, self.args) + self.rules = self.rules_loader.load(self.conf, self.args) self.writeback_index = self.conf['writeback_index'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] @@ -144,7 +145,7 @@ def __init__(self, args): self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} - self.rule_hashes = self.conf['rules_loader'].get_hashes(self.conf, self.args.rule) + self.rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) @@ -998,8 +999,7 @@ def modify_rule_for_ES5(new_rule): def load_rule_changes(self): """ Using the modification times of rule config files, syncs the running rules to match the files in rules_folder by removing, adding or reloading rules. """ - rules_loader = self.conf['rules_loader'] - new_rule_hashes = rules_loader.get_hashes(self.conf, self.args.rule) + new_rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) # Check each current rule for changes for rule_file, hash_value in self.rule_hashes.iteritems(): @@ -1011,7 +1011,7 @@ def load_rule_changes(self): if hash_value != new_rule_hashes[rule_file]: # Rule file was changed, reload rule try: - new_rule = rules_loader.load_configuration(rule_file, self.conf) + new_rule = self.rules_loader.load_configuration(rule_file, self.conf) if 'is_enabled' in new_rule and not new_rule['is_enabled']: elastalert_logger.info('Rule file %s is now disabled.' % (rule_file)) # Remove this rule if it's been disabled @@ -1022,7 +1022,7 @@ def load_rule_changes(self): self.handle_error(message) # Want to send email to address specified in the rule. Try and load the YAML to find it. try: - rule_yaml = rules_loader.load_yaml(rule_file) + rule_yaml = self.rules_loader.load_yaml(rule_file) except EAException: self.send_notification_email(exception=e) continue @@ -1048,7 +1048,7 @@ def load_rule_changes(self): if not self.args.rule: for rule_file in set(new_rule_hashes.keys()) - set(self.rule_hashes.keys()): try: - new_rule = rules_loader.load_configuration(rule_file, self.conf) + new_rule = self.rules_loader.load_configuration(rule_file, self.conf) if 'is_enabled' in new_rule and not new_rule['is_enabled']: continue if new_rule['name'] in [rule['name'] for rule in self.rules]: diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 79b7f6038..ee936839f 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -207,7 +207,6 @@ def run_elastalert(self, rule, conf, args): load_modules_args = argparse.Namespace() load_modules_args.debug = not args.alert conf['rules_loader'].load_modules(rule, load_modules_args) - conf['rules'] = [rule] # If using mock data, make sure it's sorted and find appropriate time range timestamp_field = rule.get('timestamp_field', '@timestamp') @@ -264,13 +263,15 @@ def get_id(): conf['run_every'] = endtime - starttime # Instantiate ElastAlert to use mock config and special rule - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: - load_conf.return_value = conf - if args.alert: - client = ElastAlerter(['--verbose']) - else: - client = ElastAlerter(['--debug']) + with mock.patch.object(conf['rules_loader'], 'get_hashes'): + with mock.patch.object(conf['rules_loader'], 'load') as load_rules: + load_rules.return_value = [rule] + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + load_conf.return_value = conf + if args.alert: + client = ElastAlerter(['--verbose']) + else: + client = ElastAlerter(['--debug']) # Replace get_hits_* functions to use mock data if args.json: @@ -341,23 +342,20 @@ def run_rule_test(self): parser.add_argument('--config', action='store', dest='config', help='Global config file.') args = parser.parse_args() - # rule_yaml = load_rule_yaml(args.file) - - # conf = self.load_conf(rule_yaml, args) - overrides = { + defaults = { 'rules_folder': 'rules', 'es_host': 'localhost', 'es_port': 14900, 'writeback_index': 'wb', 'max_query_size': 10000, - 'alert_time_limit': datetime.timedelta(hours=24), - 'old_query_limit': datetime.timedelta(weeks=1), - 'run_every': datetime.timedelta(minutes=5), + 'alert_time_limit': {'hours': 24}, + 'old_query_limit': {'weeks': 1}, + 'run_every': {'minutes': 5}, 'disable_rules_on_error': False, - 'buffer_time': datetime.timedelta(minutes=45), + 'buffer_time': {'minutes': 45}, 'scroll_keepalive': '30s' } - conf = load_conf(args, overrides) + conf = load_conf(args, defaults) rule_yaml = conf['rules_loader'].get_yaml(args.file) conf['rules_loader'].load_options(rule_yaml, conf, args.file) From 55e71b50f6e5b3aabd4cf3f5d315b4d3617b4be7 Mon Sep 17 00:00:00 2001 From: Ryan Fang Date: Fri, 13 Jul 2018 09:58:16 -0700 Subject: [PATCH 025/264] fixed the failed tests --- elastalert/alerts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index f5f496466..42532aa1e 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -259,6 +259,7 @@ def get_aggregation_summary_text(self, matches): text = '' if 'aggregation' in self.rule and 'summary_table_fields' in self.rule: text = self.rule.get('summary_prefix', '') + summary_table_fields = self.rule['summary_table_fields'] if not isinstance(summary_table_fields, list): summary_table_fields = [summary_table_fields] # Include a count aggregation so that we can see at a glance how many of each aggregation_key were encountered From 295bb041534baa856ec5d80533f812ee1058da0f Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 17 Jul 2018 17:43:07 -0700 Subject: [PATCH 026/264] Fix num_hits for aggregation queries --- elastalert/elastalert.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 721523776..32e00167f 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -865,6 +865,7 @@ def run_rule(self, rule, endtime, starttime=None): if rule.get('aggregation_query_element'): if endtime - tmp_endtime == segment_size: self.run_query(rule, tmp_endtime, endtime) + self.cumulative_hits += self.num_hits elif total_seconds(rule['original_starttime'] - tmp_endtime) == 0: rule['starttime'] = rule['original_starttime'] return 0 @@ -873,6 +874,7 @@ def run_rule(self, rule, endtime, starttime=None): else: if not self.run_query(rule, rule['starttime'], endtime): return 0 + self.cumulative_hits += self.num_hits rule['type'].garbage_collect(endtime) # Process any new matches From 515b15a10fc2eb8a7df26d889d044225056dd869 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Wed, 18 Jul 2018 10:31:38 +0100 Subject: [PATCH 027/264] Added ability for test rule to override loader to use file loader --- docs/source/recipes/adding_loaders.rst | 15 ++++++++------- elastalert/config.py | 8 ++++++-- elastalert/test_rule.py | 5 ++++- 3 files changed, 18 insertions(+), 10 deletions(-) diff --git a/docs/source/recipes/adding_loaders.rst b/docs/source/recipes/adding_loaders.rst index c77520551..672d42390 100644 --- a/docs/source/recipes/adding_loaders.rst +++ b/docs/source/recipes/adding_loaders.rst @@ -38,12 +38,14 @@ Now, in a file named ``mongo_loader.py``, add from pymongo import MongoClient from elastalert.loaders import RulesLoader + import yaml class MongoRulesLoader(RulesLoader): def __init__(self, conf): super(MongoRulesLoader, self).__init__(conf) self.client = MongoClient(conf['mongo_url']) - self.db = client[conf['mongo_db']] + self.db = self.client[conf['mongo_db']] + self.cache = {} def get_names(self, conf, use_rule=None): if use_rule: @@ -52,12 +54,11 @@ Now, in a file named ``mongo_loader.py``, add rules = [] self.cache = {} for rule in self.db.rules.find(): - self.cache[rule.name] = rule.yaml - rules.append(rule.yaml) + self.cache[rule['name']] = yaml.load(rule['yaml']) + rules.append(rule['name']) return rules - def get_hashes(self, conf, use_rule=None): if use_rule: return [use_rule] @@ -65,8 +66,8 @@ Now, in a file named ``mongo_loader.py``, add hashes = {} self.cache = {} for rule in self.db.rules.find(): - self.cache[rule.name] = rule.yaml - hashes[rule.name] = rule.hash + self.cache[rule['name']] = rule['yaml'] + hashes[rule['name']] = rule['hash'] return hashes @@ -74,7 +75,7 @@ Now, in a file named ``mongo_loader.py``, add if rule in self.cache: return self.cache[rule] - self.cache[rule] = self.db.rules.find_one({'name': rule}).yaml + self.cache[rule] = yaml.load(self.db.rules.find_one({'name': rule})['yaml']) return self.cache[rule] Finally, you need to specify in your ElastAlert configuration file that MongoRulesLoader should be used instead of the diff --git a/elastalert/config.py b/elastalert/config.py index 6d803fc60..7587eafef 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -27,12 +27,13 @@ } -def load_conf(args, defaults=None): +def load_conf(args, defaults=None, overwrites=None): """ Creates a conf dictionary for ElastAlerter. Loads the global config file and then each rule found in rules_folder. :param args: The parsed arguments to ElastAlert - :param defaults: Dictionary of conf values to override + :param defaults: Dictionary of default conf values + :param overwrites: Dictionary of conf values to override :return: The global configuration, a dictionary. """ filename = args.config @@ -47,6 +48,9 @@ def load_conf(args, defaults=None): if key not in conf: conf[key] = value + for key, value in (overwrites.iteritems() if overwrites is not None else []): + conf[key] = value + # Make sure we have all required globals if required_globals - frozenset(conf.keys()): raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index ee936839f..f93eee4f6 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -355,7 +355,10 @@ def run_rule_test(self): 'buffer_time': {'minutes': 45}, 'scroll_keepalive': '30s' } - conf = load_conf(args, defaults) + overwrites = { + 'rules_loader': 'file', + } + conf = load_conf(args, defaults, overwrites) rule_yaml = conf['rules_loader'].get_yaml(args.file) conf['rules_loader'].load_options(rule_yaml, conf, args.file) From cc928ae18f30058c8195abd5eddd04a52ea3fa6a Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Jul 2018 10:23:32 -0400 Subject: [PATCH 028/264] Fixing json.dumps indent The json.dumps indent parameter is a number. When switched from simplejson, the same arguments were passed, even though a different API. --- elastalert/test_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 94215bdac..0079f80a5 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -403,7 +403,7 @@ def run_rule_test(self): with open(args.save, 'wb') as data_file: # Add _id to _source for dump [doc['_source'].update({'_id': doc['_id']}) for doc in hits] - data_file.write(json.dumps([doc['_source'] for doc in hits], indent=' ')) + data_file.write(json.dumps([doc['_source'] for doc in hits], indent=1)) if args.use_downloaded: if hits: args.json = args.save From 601640ca6d0e1d1498eab9e834af8e3808aeaea4 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Jul 2018 10:27:41 -0400 Subject: [PATCH 029/264] Update test_rule.py --- elastalert/test_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 0079f80a5..8890424db 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -403,7 +403,7 @@ def run_rule_test(self): with open(args.save, 'wb') as data_file: # Add _id to _source for dump [doc['_source'].update({'_id': doc['_id']}) for doc in hits] - data_file.write(json.dumps([doc['_source'] for doc in hits], indent=1)) + data_file.write(json.dumps([doc['_source'] for doc in hits], indent=4)) if args.use_downloaded: if hits: args.json = args.save From 86178699dbdd2ef737a927e8e7a5f3bff1e3c0d9 Mon Sep 17 00:00:00 2001 From: dushujun Date: Thu, 26 Jul 2018 14:55:28 +0800 Subject: [PATCH 030/264] Fix missed to_ts_func param in ElastAlerter.get_query --- elastalert/test_rule.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 8890424db..a7e56fc50 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -70,6 +70,7 @@ def test_file(self, conf, args): starttime=start_time, endtime=end_time, timestamp_field=ts, + to_ts_func=conf['dt_to_ts'], five=conf['five'] ) index = ElastAlerter.get_index(conf, start_time, end_time) @@ -96,6 +97,7 @@ def test_file(self, conf, args): starttime=start_time, endtime=end_time, timestamp_field=ts, + to_ts_func=conf['dt_to_ts'], sort=False, five=conf['five'] ) From e70ba9625bb6f9f6a8a048128d8aab63c0027838 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 26 Jul 2018 10:37:20 -0700 Subject: [PATCH 031/264] Fixed bug where ES_USERNAME couldn't be used via env variables --- elastalert/util.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/elastalert/util.py b/elastalert/util.py index 04e87baff..33f0b4e71 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -326,9 +326,12 @@ def build_es_conn_config(conf): parsed_conf['es_conn_timeout'] = conf.get('es_conn_timeout', 20) parsed_conf['send_get_body_as'] = conf.get('es_send_get_body_as', 'GET') - if 'es_username' in conf: - parsed_conf['es_username'] = os.environ.get('ES_USERNAME', conf['es_username']) - parsed_conf['es_password'] = os.environ.get('ES_PASSWORD', conf['es_password']) + if os.environ.get('ES_USERNAME'): + parsed_conf['es_username'] = os.environ.get('ES_USERNAME') + parsed_conf['es_password'] = os.environ.get('ES_PASSWORD') + elif 'es_username' in conf: + parsed_conf['es_username'] = conf['es_username'] + parsed_conf['es_password'] = conf['es_password'] if 'aws_region' in conf: parsed_conf['aws_region'] = conf['aws_region'] From 2715debf69e12fc99ed5ac7928af38e7ec4b5865 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 31 Jul 2018 17:10:46 -0700 Subject: [PATCH 032/264] Fixed tests --- tests/alerts_test.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 3271b511b..cf46a4ba2 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -967,7 +967,8 @@ def test_slack_uses_custom_title(): rule['slack_webhook_url'], data=mock.ANY, headers={'content-type': 'application/json'}, - proxies=None + proxies=None, + verify=True ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1002,13 +1003,14 @@ def test_slack_uses_rule_name_when_custom_title_is_not_provided(): } ], 'text': '', - 'parse': 'none' + 'parse': 'none', } mock_post_request.assert_called_once_with( rule['slack_webhook_url'][0], data=mock.ANY, headers={'content-type': 'application/json'}, - proxies=None + proxies=None, + verify=True ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1044,13 +1046,14 @@ def test_slack_uses_custom_slack_channel(): } ], 'text': '', - 'parse': 'none' + 'parse': 'none', } mock_post_request.assert_called_once_with( rule['slack_webhook_url'][0], data=mock.ANY, headers={'content-type': 'application/json'}, - proxies=None + proxies=None, + verify=True ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) From aee4df1c1bbbac0e02c4a2754a4a32c7f6669642 Mon Sep 17 00:00:00 2001 From: Nate Brown Date: Sat, 4 Aug 2018 17:54:13 -0700 Subject: [PATCH 033/264] Raise an exception on parse failure --- elastalert/elastalert.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 32e00167f..4456bcbb8 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -390,6 +390,12 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): **extra_args ) self.total_hits = int(res['hits']['total']) + + if len(res['_shards']['failures']) > 0: + errs = [ e['reason']['reason'] for e in res['_shards']['failures'] if 'Failed to parse' in e['reason']['reason']] + if len(errs): + raise ElasticsearchException(errs) + logging.debug(str(res)) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages From 3515c1aa25565ba8c6f90997d8fcb007df176ea8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=B6rn=20Zaefferer?= Date: Thu, 9 Aug 2018 17:16:10 +0200 Subject: [PATCH 034/264] setup: Exclude jsonschema 3 alpha jsonschema recently released 3.0.0 alphas: https://github.com/Julian/jsonschema/releases Those break the installation, so limit the range to <3.0.0 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 38562eec6..a917dfb2a 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ 'envparse>=0.2.0', 'exotel>=0.1.3', 'jira>=1.0.10,<1.0.15', - 'jsonschema>=2.6.0', + 'jsonschema>=2.6.0,<3.0.0', 'mock>=2.0.0', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', From aabffdaa18aa1d11a22a8b8c76cbef185bb6f3ee Mon Sep 17 00:00:00 2001 From: Lorenz Bausch Date: Mon, 13 Aug 2018 13:52:17 +0200 Subject: [PATCH 035/264] Fix typo --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index df8934a18..2b1e52fdb 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1879,7 +1879,7 @@ Example usage using old-style format:: HTTP POST ~~~~~~~~~ -This alert type will send results to a JSON endpoint using HTTP POST. The key names are configurable so this is compatible with almost any endpoint. By default, the JSON will contain al the items from the match, unless you specify http_post_payload, in which case it will only contain those items. +This alert type will send results to a JSON endpoint using HTTP POST. The key names are configurable so this is compatible with almost any endpoint. By default, the JSON will contain all the items from the match, unless you specify http_post_payload, in which case it will only contain those items. Required: From bcf38969a967ee9c7f01a9e8873bc3e581017768 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 15 Aug 2018 13:35:54 -0700 Subject: [PATCH 036/264] Fix an issue with nested timestamps in Spike and Cardinality rule types --- elastalert/ruletypes.py | 15 ++++++++------- 1 file changed, 8 insertions(+), 7 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 1a53fd87d..b2aa3c076 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -435,7 +435,7 @@ def clear_windows(self, qk, event): # Reset the state and prevent alerts until windows filled again self.ref_windows[qk].clear() self.first_event.pop(qk) - self.skip_checks[qk] = event[self.ts_field] + self.rules['timeframe'] * 2 + self.skip_checks[qk] = lookup_es_key(event, self.ts_field) + self.rules['timeframe'] * 2 def handle_event(self, event, count, qk='all'): self.first_event.setdefault(qk, event) @@ -446,7 +446,7 @@ def handle_event(self, event, count, qk='all'): self.cur_windows[qk].append((event, count)) # Don't alert if ref window has not yet been filled for this key AND - if event[self.ts_field] - self.first_event[qk][self.ts_field] < self.rules['timeframe'] * 2: + if lookup_es_key(event, self.ts_field) - self.first_event[qk][self.ts_field] < self.rules['timeframe'] * 2: # ElastAlert has not been running long enough for any alerts OR if not self.ref_window_filled_once: return @@ -454,7 +454,7 @@ def handle_event(self, event, count, qk='all'): if not (self.rules.get('query_key') and self.rules.get('alert_on_new_data')): return # An alert for this qk has recently fired - if qk in self.skip_checks and event[self.ts_field] < self.skip_checks[qk]: + if qk in self.skip_checks and lookup_es_key(event, self.ts_field) < self.skip_checks[qk]: return else: self.ref_window_filled_once = True @@ -916,22 +916,23 @@ def add_data(self, data): # If no query_key, we use the key 'all' for all events key = 'all' self.cardinality_cache.setdefault(key, {}) - self.first_event.setdefault(key, event[self.ts_field]) + self.first_event.setdefault(key, lookup_es_key(event, self.ts_field)) value = hashable(lookup_es_key(event, self.cardinality_field)) if value is not None: # Store this timestamp as most recent occurence of the term - self.cardinality_cache[key][value] = event[self.ts_field] + self.cardinality_cache[key][value] = lookup_es_key(event, self.ts_field) self.check_for_match(key, event) def check_for_match(self, key, event, gc=True): # Check to see if we are past max/min_cardinality for a given key - timeframe_elapsed = event[self.ts_field] - self.first_event.get(key, event[self.ts_field]) > self.timeframe + time_elapsed = lookup_es_key(event, self.ts_field) - self.first_event.get(key, lookup_es_key(event, self.ts_field)) + timeframe_elapsed = time_elapsed > self.timeframe if (len(self.cardinality_cache[key]) > self.rules.get('max_cardinality', float('inf')) or (len(self.cardinality_cache[key]) < self.rules.get('min_cardinality', float('-inf')) and timeframe_elapsed)): # If there might be a match, run garbage collect first, as outdated terms are only removed in GC # Only run it if there might be a match so it doesn't impact performance if gc: - self.garbage_collect(event[self.ts_field]) + self.garbage_collect(lookup_es_key(event, self.ts_field)) self.check_for_match(key, event, False) else: self.first_event.pop(key, None) From 0faeaa6843e2cebabac2859750b331c93a6d5b86 Mon Sep 17 00:00:00 2001 From: "APPSON\\hossein.t" Date: Mon, 27 Aug 2018 15:07:04 +0430 Subject: [PATCH 037/264] Add MattermostAlerter --- elastalert/alerts.py | 109 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 15d05b66c..2087b01b8 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1188,6 +1188,115 @@ def get_info(self): 'slack_webhook_url': self.slack_webhook_url} +class MattermostAlerter(Alerter): + """ Creates a Mattermsot post for each alert """ + required_options = frozenset(['mattermost_webhook_url']) + + def __init__(self, rule): + super(MattermostAlerter, self).__init__(rule) + + # HTTP config + self.mattermost_webhook_url = self.rule['mattermost_webhook_url'] + if isinstance(self.mattermost_webhook_url, basestring): + self.mattermost_webhook_url = [self.mattermost_webhook_url] + self.mattermost_proxy = self.rule.get('mattermost_proxy', None) + self.mattermost_ignore_ssl_errors = self.rule.get('mattermost_ignore_ssl_errors', False) + + # Override webhook config + self.mattermost_username_override = self.rule.get('mattermost_username_override', 'elastalert') + self.mattermost_channel_override = self.rule.get('mattermost_channel_override', '') + self.mattermost_icon_url_override = self.rule.get('mattermost_icon_url_override', '') + + # Message properties + self.mattermost_msg_pretext = self.rule.get('mattermost_msg_pretext', '') + self.mattermost_msg_color = self.rule.get('mattermost_msg_color', 'danger') + self.mattermost_msg_fields = self.rule.get('mattermost_msg_fields', '') + + def get_aggregation_summary_text__maximum_width(self): + width = super(MattermostAlerter, self).get_aggregation_summary_text__maximum_width() + # Reduced maximum width for prettier Mattermost display. + return min(width, 75) + + def get_aggregation_summary_text(self, matches): + text = super(MattermostAlerter, self).get_aggregation_summary_text(matches) + if text: + text = u'```\n{0}```\n'.format(text) + return text + + def populate_fields(self, matches): + alert_fields = [] + missing = self.rule.get('alert_missing_value', '') + for field in self.mattermost_msg_fields: + field = copy.copy(field) + if 'args' in field: + args_values = [lookup_es_key(matches[0], arg) or missing for arg in field['args']] + if 'value' in field: + field['value'] = field['value'].format(*args_values) + else: + field['value'] = "\n".join(str(arg) for arg in args_values) + del(field['args']) + alert_fields.append(field) + return alert_fields + + def alert(self, matches): + body = self.create_alert_body(matches) + title = self.create_title(matches) + + # post to mattermost + headers = {'content-type': 'application/json'} + # set https proxy, if it was provided + proxies = {'https': self.mattermost_proxy} if self.mattermost_proxy else None + payload = { + 'attachments': [ + { + 'fallback': "{0}: {1}".format(title, self.mattermost_msg_pretext), + 'color': self.mattermost_msg_color, + 'title': title, + 'pretext': self.mattermost_msg_pretext, + 'fields': [] + } + ] + } + + if self.rule.get('alert_text_type') == 'alert_text_only': + payload['attachments'][0]['text'] = body + else: + payload['text'] = body + + if self.mattermost_msg_fields != '': + payload['attachments'][0]['fields'] = self.populate_fields(matches) + + if self.mattermost_icon_url_override != '': + payload['icon_url'] = self.mattermost_icon_url_override + + if self.mattermost_username_override != '': + payload['username'] = self.mattermost_username_override + + if self.mattermost_channel_override != '': + payload['channel'] = self.mattermost_channel_override + + for url in self.mattermost_webhook_url: + try: + if self.mattermost_ignore_ssl_errors: + requests.urllib3.disable_warnings() + + response = requests.post( + url, data=json.dumps(payload, cls=DateTimeEncoder), + headers=headers, verify=not self.mattermost_ignore_ssl_errors, + proxies=proxies) + + warnings.resetwarnings() + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to Mattermost: %s" % e) + elastalert_logger.info("Alert sent to Mattermost") + + def get_info(self): + return {'type': 'mattermost', + 'mattermost_username_override': self.mattermost_username_override, + 'mattermost_webhook_url': self.mattermost_webhook_url} + + class PagerDutyAlerter(Alerter): """ Create an incident on PagerDuty for each alert """ required_options = frozenset(['pagerduty_service_key', 'pagerduty_client_name']) From 2a98a953f1dfe5a84ab8d104640f65b90b8f69e2 Mon Sep 17 00:00:00 2001 From: "APPSON\\hossein.t" Date: Mon, 27 Aug 2018 15:18:12 +0430 Subject: [PATCH 038/264] Add schema for mattermost config --- elastalert/schema.yaml | 20 ++++++++++++++++++++ 1 file changed, 20 insertions(+) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 4afad9f1f..617cfae03 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -25,6 +25,15 @@ definitions: filter: &filter {} + mattermostField: &mattermostField + type: object + additionalProperties: false + properties: + title: {type: string} + value: {type: string} + args: *arrayOfString + short: {type: boolean} + required: [type, index, alert] type: object @@ -239,6 +248,17 @@ properties: slack_text_string: {type: string} slack_ignore_ssl_errors: {type: boolean} + ### Mattermost + mattermost_webhook_url: *arrayOfString + mattermost_proxy: {type: string} + mattermost_ignore_ssl_errors: {type: boolean} + mattermost_username_override: {type: string} + mattermost_icon_url_override: {type: string} + mattermost_channel_override: {type: string} + mattermost_msg_color: {enum: [good, warning, danger]} + mattermost_msg_pretext: {type: string} + mattermost_msg_fields: *mattermostField + ### PagerDuty pagerduty_service_key: {type: string} pagerduty_client_name: {type: string} From d5dce9bd972d3876fc845e28537de9429776195c Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 27 Aug 2018 11:04:28 -0700 Subject: [PATCH 039/264] Fixed tests --- elastalert/elastalert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 4456bcbb8..f9047d31d 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -391,8 +391,8 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): ) self.total_hits = int(res['hits']['total']) - if len(res['_shards']['failures']) > 0: - errs = [ e['reason']['reason'] for e in res['_shards']['failures'] if 'Failed to parse' in e['reason']['reason']] + if len(res.get('_shards', {}).get('failures', [])) > 0: + errs = [e['reason']['reason'] for e in res['_shards']['failures'] if 'Failed to parse' in e['reason']['reason']] if len(errs): raise ElasticsearchException(errs) From bde593c342e4a52397a9db58f0b7366ae8fca167 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 27 Aug 2018 11:10:07 -0700 Subject: [PATCH 040/264] Version 0.1.34 --- changelog.md | 16 ++++++++++++++++ setup.py | 2 +- 2 files changed, 17 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 8dde55dfb..22c780b61 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,21 @@ # Change Log +# v0.1.34 + +### Added +- Added prefix/suffix support for summary table +- Added support for ignoring SSL validation in Slack +- More visible exceptions during query parse failures + +### Fixed +- Fixed top_count_keys when using compound query_key +- Fixed num_hits sometimes being reported too low +- Fixed an issue with setting ES_USERNAME via env +- Fixed an issue when using test script with custom timestamps +- Fixed a unicode error when using Telegram +- Fixed an issue with jsonschema version conflict +- Fixed an issue with nested timestamps in cardinality type + # v0.1.33 ### Added diff --git a/setup.py b/setup.py index a917dfb2a..966ddd78f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.33', + version='0.1.34', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From 6ecaa0cbf3ebab41b21e568f0323a38791777231 Mon Sep 17 00:00:00 2001 From: "APPSON\\hossein.t" Date: Tue, 28 Aug 2018 15:10:24 +0430 Subject: [PATCH 041/264] Add mattermost to alerts mapping --- elastalert/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/config.py b/elastalert/config.py index c6efb3ad2..ac4c58ab0 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -73,6 +73,7 @@ 'stride': alerts.StrideAlerter, 'ms_teams': alerts.MsTeamsAlerter, 'slack': alerts.SlackAlerter, + 'mattermost': alerts.MattermostAlerter, 'pagerduty': alerts.PagerDutyAlerter, 'exotel': alerts.ExotelAlerter, 'twilio': alerts.TwilioAlerter, From 569b51effe0fbea432e3b7f05d2074c27bf10260 Mon Sep 17 00:00:00 2001 From: "APPSON\\hossein.t" Date: Tue, 28 Aug 2018 15:34:48 +0430 Subject: [PATCH 042/264] Remove white space --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 2087b01b8..5da6ed0ab 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1260,7 +1260,7 @@ def alert(self, matches): if self.rule.get('alert_text_type') == 'alert_text_only': payload['attachments'][0]['text'] = body - else: + else: payload['text'] = body if self.mattermost_msg_fields != '': From d128d8e4e197452157874f56142bd222e9ddfe1d Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 28 Aug 2018 10:57:03 -0700 Subject: [PATCH 043/264] Fixed an issue preventing new_term rule from working with use_terms_query --- elastalert/ruletypes.py | 2 +- elastalert/schema.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index b2aa3c076..d3511a582 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -639,7 +639,7 @@ def __init__(self, rule, args=None): (len(self.fields) != 1 or (len(self.fields) == 1 and type(self.fields[0]) == list)): raise EAException("use_terms_query can only be used with a single non-composite field") if self.rules.get('use_terms_query'): - if self.rules.get('query_key') != self.fields: + if [self.rules['query_key']] != self.fields: raise EAException('If use_terms_query is specified, you cannot specify different query_key and fields') if not self.rules.get('query_key').endswith('.keyword') and not self.rules.get('query_key').endswith('.raw'): if self.rules.get('use_keyword_postfix', True): diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 4afad9f1f..c3d4509e9 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -94,7 +94,7 @@ oneOf: doc_type: {type: string} - title: New Term - required: [fields] + required: [] properties: type: {enum: [new_term]} fields: *arrayOfStringsOrOtherArray From 7acd800d82e9de7ee06851d10e4acf7b173e3cac Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 28 Aug 2018 10:58:52 -0700 Subject: [PATCH 044/264] Version 0.1.35 --- changelog.md | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 22c780b61..176eda54a 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,10 @@ # Change Log +# v0.1.35 + +### Fixed +- Fixed an issue preventing new term rule from working with terms query + # v0.1.34 ### Added diff --git a/setup.py b/setup.py index 966ddd78f..5f0d99732 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.34', + version='0.1.35', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From ee6bf1ebd1a8b31c7e0fdecbe610e1026b08a58b Mon Sep 17 00:00:00 2001 From: Aleksandr Makhinov Date: Thu, 30 Aug 2018 11:00:46 +0300 Subject: [PATCH 045/264] Add auth proxy support for telegram --- elastalert/alerts.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 15d05b66c..7a0f765ce 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -23,6 +23,7 @@ from exotel import Exotel from jira.client import JIRA from jira.exceptions import JIRAError +from requests.auth import HTTPProxyAuth from requests.exceptions import RequestException from staticconf.loader import yaml_loader from texttable import Texttable @@ -1399,6 +1400,8 @@ def __init__(self, rule): self.telegram_api_url = self.rule.get('telegram_api_url', 'api.telegram.org') self.url = 'https://%s/bot%s/%s' % (self.telegram_api_url, self.telegram_bot_token, "sendMessage") self.telegram_proxy = self.rule.get('telegram_proxy', None) + self.telegram_proxy_login = self.rule.get('telegram_proxy_login', None) + self.telegram_proxy_password = self.rule.get('telegram_proxy_pass', None) def alert(self, matches): body = u'⚠ *%s* ⚠ ```\n' % (self.create_title(matches)) @@ -1414,6 +1417,7 @@ def alert(self, matches): headers = {'content-type': 'application/json'} # set https proxy, if it was provided proxies = {'https': self.telegram_proxy} if self.telegram_proxy else None + auth = HTTPProxyAuth(self.telegram_proxy_login,self.telegram_proxy_password) if self.telegram_proxy_login else None payload = { 'chat_id': self.telegram_room_id, 'text': body, @@ -1422,7 +1426,7 @@ def alert(self, matches): } try: - response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies) + response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies, auth=auth) warnings.resetwarnings() response.raise_for_status() except RequestException as e: From 70abbc582be262835544dcb58e9a5b29d84888bc Mon Sep 17 00:00:00 2001 From: Mike Rohland Date: Wed, 29 Aug 2018 12:59:05 +0200 Subject: [PATCH 046/264] fix username,name for responders in opsgenie alert v2 api --- elastalert/opsgenie.py | 7 ++----- tests/alerts_test.py | 4 ++-- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index 45bf27dd1..fa72a23f6 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -29,9 +29,6 @@ def __init__(self, *args): self.opsgenie_proxy = self.rule.get('opsgenie_proxy', None) self.priority = self.rule.get('opsgenie_priority') - def _fill_responders(self, responders, type_): - return [{'id': r, 'type': type_} for r in responders] - def alert(self, matches): body = '' for match in matches: @@ -50,9 +47,9 @@ def alert(self, matches): if self.account: post['user'] = self.account if self.recipients: - post['responders'] = self._fill_responders(self.recipients, 'user') + post['responders'] = [{'username': r, 'type': 'user'} for r in self.recipients] if self.teams: - post['teams'] = self._fill_responders(self.teams, 'team') + post['teams'] = [{'name': r, 'type': 'team'} for r in self.teams] post['description'] = body post['source'] = 'ElastAlert' post['tags'] = self.tags diff --git a/tests/alerts_test.py b/tests/alerts_test.py index cf46a4ba2..f03a67063 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -389,7 +389,7 @@ def test_opsgenie_basic(): assert mcal[0][1]['headers']['Authorization'] == 'GenieKey ogkey' assert mcal[0][1]['json']['source'] == 'ElastAlert' - assert mcal[0][1]['json']['responders'] == [{'id': 'lytics', 'type': 'user'}] + assert mcal[0][1]['json']['responders'] == [{'username': 'lytics', 'type': 'user'}] assert mcal[0][1]['json']['source'] == 'ElastAlert' @@ -415,7 +415,7 @@ def test_opsgenie_frequency(): assert mcal[0][1]['headers']['Authorization'] == 'GenieKey ogkey' assert mcal[0][1]['json']['source'] == 'ElastAlert' - assert mcal[0][1]['json']['responders'] == [{'id': 'lytics', 'type': 'user'}] + assert mcal[0][1]['json']['responders'] == [{'username': 'lytics', 'type': 'user'}] assert mcal[0][1]['json']['source'] == 'ElastAlert' assert mcal[0][1]['json']['source'] == 'ElastAlert' From 2fd1c81b5dffa59e795c3c2ac622c9a9677019e1 Mon Sep 17 00:00:00 2001 From: Mike Rohland Date: Wed, 1 Aug 2018 14:23:05 +0200 Subject: [PATCH 047/264] allow list of slack channel overrides additionally to list of webhooks --- elastalert/alerts.py | 29 +++++++++++---------- tests/alerts_test.py | 60 ++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 76 insertions(+), 13 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 15d05b66c..e0eb86f44 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1104,6 +1104,8 @@ def __init__(self, rule): self.slack_proxy = self.rule.get('slack_proxy', None) self.slack_username_override = self.rule.get('slack_username_override', 'elastalert') self.slack_channel_override = self.rule.get('slack_channel_override', '') + if isinstance(self.slack_channel_override, basestring): + self.slack_channel_override = [self.slack_channel_override] self.slack_emoji_override = self.rule.get('slack_emoji_override', ':ghost:') self.slack_icon_url_override = self.rule.get('slack_icon_url_override', '') self.slack_msg_color = self.rule.get('slack_msg_color', 'danger') @@ -1145,7 +1147,6 @@ def alert(self, matches): proxies = {'https': self.slack_proxy} if self.slack_proxy else None payload = { 'username': self.slack_username_override, - 'channel': self.slack_channel_override, 'parse': self.slack_parse_override, 'text': self.slack_text_string, 'attachments': [ @@ -1169,18 +1170,20 @@ def alert(self, matches): payload['icon_emoji'] = self.slack_emoji_override for url in self.slack_webhook_url: - try: - if self.slack_ignore_ssl_errors: - requests.packages.urllib3.disable_warnings() - response = requests.post( - url, data=json.dumps(payload, cls=DateTimeEncoder), - headers=headers, verify=not self.slack_ignore_ssl_errors, - proxies=proxies) - warnings.resetwarnings() - response.raise_for_status() - except RequestException as e: - raise EAException("Error posting to slack: %s" % e) - elastalert_logger.info("Alert sent to Slack") + for channel_override in self.slack_channel_override: + try: + if self.slack_ignore_ssl_errors: + requests.packages.urllib3.disable_warnings() + payload['channel'] = channel_override + response = requests.post( + url, data=json.dumps(payload, cls=DateTimeEncoder), + headers=headers, verify=not self.slack_ignore_ssl_errors, + proxies=proxies) + warnings.resetwarnings() + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to slack: %s" % e) + elastalert_logger.info("Alert '%s' sent to Slack" % self.rule['name']) def get_info(self): return {'type': 'slack', diff --git a/tests/alerts_test.py b/tests/alerts_test.py index cf46a4ba2..ee5abcfde 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1058,6 +1058,66 @@ def test_slack_uses_custom_slack_channel(): assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) +def test_slack_uses_list_of_custom_slack_channel(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_webhook_url': ['http://please.dontgohere.slack'], + 'slack_channel_override': ['#test-alert', '#test-alert2'], + 'alert': [] + } + load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'somefield': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data1 = { + 'username': 'elastalert', + 'channel': '#test-alert', + 'icon_emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': rule['name'], + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'text': '', + 'parse': 'none' + } + expected_data2 = { + 'username': 'elastalert', + 'channel': '#test-alert2', + 'icon_emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': rule['name'], + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'text': '', + 'parse': 'none' + } + mock_post_request.assert_called_with( + rule['slack_webhook_url'][0], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True + ) + assert expected_data1 == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data2 == json.loads(mock_post_request.call_args_list[1][1]['data']) + + def test_http_alerter_with_payload(): rule = { 'name': 'Test HTTP Post Alerter With Payload', From ee95d3b20240eba43bf8bdfa2b0189e17fb1160c Mon Sep 17 00:00:00 2001 From: Laurent Butti Date: Fri, 31 Aug 2018 14:18:43 +0200 Subject: [PATCH 048/264] fix/cumulative_hits --- elastalert/elastalert.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index f9047d31d..7de726aaa 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -141,6 +141,7 @@ def __init__(self, args): self.smtp_host = self.conf.get('smtp_host', 'localhost') self.max_aggregation = self.conf.get('max_aggregation', 10000) self.alerts_sent = 0 + self.cumulative_hits = 0 self.num_hits = 0 self.num_dupes = 0 self.current_es = None From 420791841773a6608f982974ad9156690c1c1a8f Mon Sep 17 00:00:00 2001 From: Laurent Butti Date: Tue, 4 Sep 2018 14:46:44 +0200 Subject: [PATCH 049/264] slack_ca_certs from ca_certs --- elastalert/alerts.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 8eac5ece7..acbef1d42 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1114,6 +1114,7 @@ def __init__(self, rule): self.slack_text_string = self.rule.get('slack_text_string', '') self.slack_alert_fields = self.rule.get('slack_alert_fields', '') self.slack_ignore_ssl_errors = self.rule.get('slack_ignore_ssl_errors', False) + self.slack_ca_certs = self.rule.get('slack_ca_certs') def format_body(self, body): # https://api.slack.com/docs/formatting @@ -1173,12 +1174,16 @@ def alert(self, matches): for url in self.slack_webhook_url: for channel_override in self.slack_channel_override: try: + if self.slack_ca_certs: + verify = self.slack_ca_certs + else: + verify = self.slack_ignore_ssl_errors if self.slack_ignore_ssl_errors: requests.packages.urllib3.disable_warnings() payload['channel'] = channel_override response = requests.post( url, data=json.dumps(payload, cls=DateTimeEncoder), - headers=headers, verify=not self.slack_ignore_ssl_errors, + headers=headers, verify=verify, proxies=proxies) warnings.resetwarnings() response.raise_for_status() From 6d71629f6148c5e11b9de16248bb1d4b7a4f0162 Mon Sep 17 00:00:00 2001 From: Laurent Butti Date: Tue, 4 Sep 2018 14:58:46 +0200 Subject: [PATCH 050/264] updated schema.yaml --- elastalert/schema.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index c3d4509e9..bee18c1d9 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -238,6 +238,7 @@ properties: slack_parse_override: {enum: [none, full]} slack_text_string: {type: string} slack_ignore_ssl_errors: {type: boolean} + slack_ca_certs: {type: string} ### PagerDuty pagerduty_service_key: {type: string} From e653e4d694c8bb0c184a4105ee2592a02870198f Mon Sep 17 00:00:00 2001 From: John Susek Date: Tue, 11 Sep 2018 11:57:33 -0500 Subject: [PATCH 051/264] Add option "--formatted-output" to make stdout print JSON formatted result. --- elastalert/test_rule.py | 48 ++++++++++++++++++++++++++++++++++------- 1 file changed, 40 insertions(+), 8 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index a7e56fc50..4df728df0 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -43,6 +43,7 @@ def print_terms(terms, parent): class MockElastAlerter(object): def __init__(self): self.data = [] + self.formatted_output = {} def test_file(self, conf, args): """ Loads a rule config file, performs a query over the last day (args.days), lists available keys @@ -111,9 +112,15 @@ def test_file(self, conf, args): return None num_hits = res['count'] - print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else '')) - print("\nAvailable terms in first hit:") - print_terms(terms, '') + + if args.formatted_output: + self.formatted_output['hits'] = num_hits + self.formatted_output['days'] = args.days + self.formatted_output['terms'] = terms + else: + print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else '')) + print("\nAvailable terms in first hit:") + print_terms(terms, '') # Check for missing keys pk = conf.get('primary_key') @@ -133,7 +140,9 @@ def test_file(self, conf, args): # If the index starts with 'logstash', fields with .raw will be available but won't in _source if term not in terms and not (term.endswith('.raw') and term[:-4] in terms and index.startswith('logstash')): print("top_count_key %s may be missing" % (term), file=sys.stderr) - print('') # Newline + + if not args.formatted_output: + print('') # Newline # Download up to max_query_size (defaults to 10,000) documents to save if args.save and not args.count: @@ -146,7 +155,12 @@ def test_file(self, conf, args): exit(1) return None num_hits = len(res['hits']['hits']) - print("Downloaded %s documents to save" % (num_hits)) + + if args.formatted_output: + self.formatted_output['download_hits'] = num_hits + else: + print("Downloaded %s documents to save" % (num_hits)) + return res['hits']['hits'] def mock_count(self, rule, start, end, index): @@ -291,10 +305,19 @@ def get_id(): client.run_rule(rule, endtime, starttime) if mock_writeback.call_count: - print("\nWould have written the following documents to writeback index (default is elastalert_status):\n") + + if args.formatted_output: + self.formatted_output['writeback'] = {} + else: + print("\nWould have written the following documents to writeback index (default is elastalert_status):\n") + errors = False for call in mock_writeback.call_args_list: - print("%s - %s\n" % (call[0][0], call[0][1])) + if args.formatted_output: + self.formatted_output['writeback'][call[0][0]] = json.loads(json.dumps(call[0][1], default=str)) + else: + print("%s - %s\n" % (call[0][0], call[0][1])) + if call[0][0] == 'elastalert_error': errors = True if errors and args.stop_error: @@ -340,7 +363,12 @@ def load_conf(self, rules, args): conf[key] = conf_default[key] elastalert.config.base_config = copy.deepcopy(conf) load_options(rules, conf, args.file) - print("Successfully loaded %s\n" % (rules['name'])) + + if args.formatted_output: + self.formatted_output['success'] = True + self.formatted_output['name'] = rules['name'] + else: + print("Successfully loaded %s\n" % (rules['name'])) return conf @@ -356,6 +384,7 @@ def run_rule_test(self): parser.add_argument('--end', dest='end', help='YYYY-MM-DDTHH:MM:SS Query to this timestamp. (Default: present) ' 'Use "NOW" to start from current time. (Default: present)') parser.add_argument('--stop-error', action='store_true', help='Stop the entire test right after the first error') + parser.add_argument('--formatted-output', action='store_true', help='Output results in formatted JSON') parser.add_argument( '--data', type=str, @@ -417,6 +446,9 @@ def run_rule_test(self): if not args.schema_only and not args.count: self.run_elastalert(rule_yaml, conf, args) + if args.formatted_output: + print(json.dumps(self.formatted_output)) + def main(): test_instance = MockElastAlerter() From b518cd096ce6de27952929ada313826e2f9adbb1 Mon Sep 17 00:00:00 2001 From: John Susek Date: Tue, 11 Sep 2018 12:24:02 -0500 Subject: [PATCH 052/264] Tweak formatting to fix travis ci fail --- elastalert/test_rule.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 4df728df0..5c668b1b8 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -140,7 +140,6 @@ def test_file(self, conf, args): # If the index starts with 'logstash', fields with .raw will be available but won't in _source if term not in terms and not (term.endswith('.raw') and term[:-4] in terms and index.startswith('logstash')): print("top_count_key %s may be missing" % (term), file=sys.stderr) - if not args.formatted_output: print('') # Newline @@ -160,7 +159,6 @@ def test_file(self, conf, args): self.formatted_output['download_hits'] = num_hits else: print("Downloaded %s documents to save" % (num_hits)) - return res['hits']['hits'] def mock_count(self, rule, start, end, index): From d5d88fb5862a0f5de40787998866dec4f515724c Mon Sep 17 00:00:00 2001 From: John Susek Date: Tue, 11 Sep 2018 12:30:28 -0500 Subject: [PATCH 053/264] Update docs. --- docs/source/ruletypes.rst | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 2b1e52fdb..5a88f9a98 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -687,6 +687,8 @@ guaranteed to have the exact same results as with Elasticsearch. For example, an ``--alert``: Trigger real alerts instead of the debug (logging text) alert. +``--formatted-output``: Output results in formatted JSON. + .. note:: Results from running this script may not always be the same as if an actual ElastAlert instance was running. Some rule types, such as spike and flatline require a minimum elapsed time before they begin alerting, based on their timeframe. In addition, use_count_query and From f97354d516b87bbec31366bd4252e0bc6d5d69f8 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 11 Sep 2018 10:34:31 -0700 Subject: [PATCH 054/264] Fixed a style issue --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 8eac5ece7..0babda00a 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1420,7 +1420,7 @@ def alert(self, matches): headers = {'content-type': 'application/json'} # set https proxy, if it was provided proxies = {'https': self.telegram_proxy} if self.telegram_proxy else None - auth = HTTPProxyAuth(self.telegram_proxy_login,self.telegram_proxy_password) if self.telegram_proxy_login else None + auth = HTTPProxyAuth(self.telegram_proxy_login, self.telegram_proxy_password) if self.telegram_proxy_login else None payload = { 'chat_id': self.telegram_room_id, 'text': body, From b1ef07fb28b57bd7516b08bd8d469551905a7756 Mon Sep 17 00:00:00 2001 From: Laurent Butti Date: Wed, 12 Sep 2018 09:00:04 +0200 Subject: [PATCH 055/264] added min_denominator feature --- docs/source/ruletypes.rst | 2 ++ elastalert/ruletypes.py | 3 ++- 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 2b1e52fdb..cec48d8bd 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1124,6 +1124,8 @@ evaluated separately against the threshold(s). For example, "%.2f" will round it to 2 decimal places. See: https://docs.python.org/3.4/library/string.html#format-specification-mini-language +``min_denominator``: Minimum number of documents on which percentage calculation will apply. Default is 0. + .. _alerts: Alerts diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index d3511a582..fe54673e0 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1109,6 +1109,7 @@ def __init__(self, *args): if 'max_percentage' not in self.rules and 'min_percentage' not in self.rules: raise EAException("PercentageMatchRule must have at least one of either min_percentage or max_percentage") + self.min_denominator = self.rules.get('min_denominator', 0) self.match_bucket_filter = self.rules['match_bucket_filter'] self.rules['aggregation_query_element'] = self.generate_aggregation_query() @@ -1146,7 +1147,7 @@ def check_matches(self, timestamp, query_key, aggregation_data): return else: total_count = other_bucket_count + match_bucket_count - if total_count == 0: + if total_count == 0 or total_count < self.min_denominator: return else: match_percentage = (match_bucket_count * 1.0) / (total_count * 1.0) * 100 From 03693d846600fd994fb497a1e284e6b228afd263 Mon Sep 17 00:00:00 2001 From: John Susek Date: Wed, 12 Sep 2018 11:00:20 -0500 Subject: [PATCH 056/264] Add results to json formatted test_rule output --- elastalert/test_rule.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 5c668b1b8..fb97579b6 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -144,7 +144,7 @@ def test_file(self, conf, args): print('') # Newline # Download up to max_query_size (defaults to 10,000) documents to save - if args.save and not args.count: + if (args.save or args.formatted_output) and not args.count : try: res = es_client.search(index, size=args.max_query_size, body=query, ignore_unavailable=True) except Exception as e: @@ -155,9 +155,7 @@ def test_file(self, conf, args): return None num_hits = len(res['hits']['hits']) - if args.formatted_output: - self.formatted_output['download_hits'] = num_hits - else: + if args.save: print("Downloaded %s documents to save" % (num_hits)) return res['hits']['hits'] @@ -428,6 +426,8 @@ def run_rule_test(self): self.data = json.loads(data_file.read()) else: hits = self.test_file(copy.deepcopy(rule_yaml), args) + if hits and args.formatted_output: + self.formatted_output['results'] = json.loads(json.dumps(hits)) if hits and args.save: with open(args.save, 'wb') as data_file: # Add _id to _source for dump From a1a6e2039d9d84a0a81b4775a3f80decf3f8fb74 Mon Sep 17 00:00:00 2001 From: John Susek Date: Wed, 12 Sep 2018 11:01:14 -0500 Subject: [PATCH 057/264] Make terms a list instead of dict in json formatted test output --- elastalert/test_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index fb97579b6..cc091a2d9 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -116,7 +116,7 @@ def test_file(self, conf, args): if args.formatted_output: self.formatted_output['hits'] = num_hits self.formatted_output['days'] = args.days - self.formatted_output['terms'] = terms + self.formatted_output['terms'] = terms.keys() else: print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else '')) print("\nAvailable terms in first hit:") From d8e7a7f0d88fcbff3f7a7860d9e10e432a54f24d Mon Sep 17 00:00:00 2001 From: John Susek Date: Wed, 12 Sep 2018 11:11:50 -0500 Subject: [PATCH 058/264] Fix travis ci fail --- elastalert/test_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index cc091a2d9..e05384816 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -144,7 +144,7 @@ def test_file(self, conf, args): print('') # Newline # Download up to max_query_size (defaults to 10,000) documents to save - if (args.save or args.formatted_output) and not args.count : + if (args.save or args.formatted_output) and not args.count: try: res = es_client.search(index, size=args.max_query_size, body=query, ignore_unavailable=True) except Exception as e: From 492ac0ccba6883fe62ac0eaf9cb5b297c243d5af Mon Sep 17 00:00:00 2001 From: Hossein Taleghani Date: Wed, 12 Sep 2018 22:10:47 +0430 Subject: [PATCH 059/264] Add docs --- docs/source/ruletypes.rst | 30 ++++++++++++++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 2b1e52fdb..b994c2d7b 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1617,6 +1617,36 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_alert_fields``: You can add additional fields to your slack alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`. +Mattermost +~~~~~ + +Mattermost alerter will send a notification to a predefined Mattermost channel. The body of the notification is formatted the same as with other alerters. + +The alerter requires the following option: + +``mattermost_webhook_url``: The webhook URL. Follow the instructions on https://docs.mattermost.com/developer/webhooks-incoming.html to create an incoming webhook on your Mattermost installation. + +Optional: + +``mattermost_proxy``: By default ElastAlert will not use a network proxy to send notifications to Mattermost. Set this option using ``hostname:port`` if you need to use a proxy. + +``mattermost_ignore_ssl_errors``: By default ElastAlert will verify SSL certificate. Set this option to ``False`` if you want to ignore SSL errors. + +``mattermost_username_override``: By default Mattermost will use your username when posting to the channel. Use this option to change it (free text). + +``mattermost_channel_override``: Incoming webhooks have a default channel, but it can be overridden. A public channel can be specified "#other-channel", and a Direct Message with "@username". + +``mattermost_icon_url_override``: By default ElastAlert will use the default webhook icon when posting to the channel. You can provide icon_url to use custom image. +Provide absolute address of the picture (for example: http://some.address.com/image.jpg) or Base64 data url. + +``mattermost_msg_pretext``: You can set the message attachment pretext using this option. + +``mattermost_msg_color``: By default the alert will be posted with the 'danger' color. You can also use 'good', 'warning', or hex color code. + +``mattermost_msg_fields``: You can add fields to your Mattermost alerts using this option. You can specify the title using `title` and the text value using `value`. Additionally you can specify whether this field should be a `short` field using `short: true`. If you set `args` and `value` is a formattable string, ElastAlert will format the incident key based on the provided array of fields from the rule or match. +See https://docs.mattermost.com/developer/message-attachments.html#fields for more information. + + Telegram ~~~~~~~~ Telegram alerter will send a notification to a predefined Telegram username or channel. The body of the notification is formatted the same as with other alerters. From dd39f42ef40a37990e3913faebf53036ef8c86c7 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 12 Sep 2018 14:48:21 -0700 Subject: [PATCH 060/264] Added a clarification on num_events --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 5a88f9a98..00b0eeb4b 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -787,7 +787,7 @@ may be counted on a per-``query_key`` basis. This rule requires two additional options: -``num_events``: The number of events which will trigger an alert. +``num_events``: The number of events which will trigger an alert, inclusive. ``timeframe``: The time that ``num_events`` must occur within. From de3ab5137cde9259126123689110c2471d506058 Mon Sep 17 00:00:00 2001 From: Alexander Nieuwenhuijse Date: Thu, 13 Sep 2018 14:59:16 +0200 Subject: [PATCH 061/264] Added Google Chat alerter --- README.md | 1 + docs/source/elastalert.rst | 1 + docs/source/ruletypes.rst | 21 +++++++++ elastalert/alerts.py | 90 ++++++++++++++++++++++++++++++++++++++ elastalert/config.py | 1 + 5 files changed, 114 insertions(+) diff --git a/README.md b/README.md index f63e25de4..0ac17a8ae 100644 --- a/README.md +++ b/README.md @@ -47,6 +47,7 @@ Currently, we have built-in support for the following alert types: - MS Teams - Slack - Telegram +- GoogleChat - AWS SNS - VictorOps - PagerDuty diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index fd2daec8c..14122b61a 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -39,6 +39,7 @@ Currently, we have support built in for these alert types: - HipChat - Slack - Telegram +- GoogleChat - Debug - Stomp diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 00b0eeb4b..35e80fe71 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1635,6 +1635,27 @@ Optional: ``telegram_proxy``: By default ElastAlert will not use a network proxy to send notifications to Telegram. Set this option using ``hostname:port`` if you need to use a proxy. +GoogleChat +~~~~~~~~~~ +GoogleChat alerter will send a notification to a predefined GoogleChat channel. The body of the notification is formatted the same as with other alerters. + +The alerter requires the following options: + +``googlechat_webhook_url``: The webhook URL that includes the channel (room) you want to post to. Go to the Google Chat website https://chat.google.com and choose the channel in which you wish to receive the notifications. Select 'Configure Webhooks' to create a new webhook or to copy the URL from an existing one. You can use a list of URLs to send to multiple channels. + +Optional: + +``googlechat_format``: Formatting for the notification. Can be either 'card' or 'basic' (default). + +``googlechat_header_title``: Sets the text for the card header title. (Only used if format=card) + +``googlechat_header_subtitle``: Sets the text for the card header subtitle. (Only used if format=card) + +``googlechat_header_image``: URL for the card header icon. (Only used if format=card) + +``googlechat_footer_kibanalink``: URL to Kibana to include in the card footer. (Only used if format=card) + + PagerDuty ~~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 0babda00a..24e0648ca 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1443,6 +1443,96 @@ def get_info(self): 'telegram_room_id': self.telegram_room_id} +class GoogleChatAlerter(Alerter): + """ Send a notification via Google Chat webhooks """ + required_options = frozenset(['googlechat_webhook_url']) + + def __init__(self, rule): + super(GoogleChatAlerter, self).__init__(rule) + self.googlechat_webhook_url = self.rule['googlechat_webhook_url'] + if isinstance(self.googlechat_webhook_url, basestring): + self.googlechat_webhook_url = [self.googlechat_webhook_url] + self.googlechat_format = self.rule.get('googlechat_format', 'basic') + self.googlechat_header_title = self.rule.get('googlechat_header_title', None) + self.googlechat_header_subtitle = self.rule.get('googlechat_header_subtitle', None) + self.googlechat_header_image = self.rule.get('googlechat_header_image', None) + self.googlechat_footer_kibanalink = self.rule.get('googlechat_footer_kibanalink', None) + + def create_header(self): + header = None + if self.googlechat_header_title: + header = { + "title": self.googlechat_header_title, + "subtitle": self.googlechat_header_subtitle, + "imageUrl": self.googlechat_header_image + } + return header + + def create_footer(self): + footer = None + if self.googlechat_footer_kibanalink: + footer = {"widgets": [{ + "buttons": [{ + "textButton": { + "text": "VISIT KIBANA", + "onClick": { + "openLink": { + "url": self.googlechat_footer_kibanalink + } + } + } + }] + }] + } + return footer + + def create_card(self, matches): + card = {"cards": [{ + "sections": [{ + "widgets": [ + {"textParagraph": {"text": self.create_alert_body(matches).encode('UTF-8')}} + ]} + ]} + ]} + + # Add the optional header + header = self.create_header() + if header: + card['cards'][0]['header'] = header + + # Add the optional footer + footer = self.create_footer() + if footer: + card['cards'][0]['sections'].append(footer) + return card + + def create_basic(self, matches): + body = self.create_alert_body(matches) + body = body.encode('UTF-8') + return {'text': body} + + def alert(self, matches): + # Format message + if self.googlechat_format == 'card': + message = self.create_card(matches) + else: + message = self.create_basic(matches) + + # Post to webhook + headers = {'content-type': 'application/json'} + for url in self.googlechat_webhook_url: + try: + response = requests.post(url, data=json.dumps(message), headers=headers) + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to google chat: {}".format(e)) + elastalert_logger.info("Alert sent to Google Chat!") + + def get_info(self): + return {'type': 'googlechat', + 'googlechat_webhook_url': self.googlechat_webhook_url} + + class GitterAlerter(Alerter): """ Creates a Gitter activity message for each alert """ required_options = frozenset(['gitter_webhook_url']) diff --git a/elastalert/config.py b/elastalert/config.py index c6efb3ad2..a36d76bda 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -78,6 +78,7 @@ 'twilio': alerts.TwilioAlerter, 'victorops': alerts.VictorOpsAlerter, 'telegram': alerts.TelegramAlerter, + 'googlechat': alerts.GoogleChatAlerter, 'gitter': alerts.GitterAlerter, 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, From cd1697b0f31e9098ac3e046dd141cafef46803ba Mon Sep 17 00:00:00 2001 From: Alexander Nieuwenhuijse Date: Thu, 13 Sep 2018 15:15:13 +0200 Subject: [PATCH 062/264] Fixed code style --- elastalert/alerts.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 24e0648ca..613602657 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1453,7 +1453,7 @@ def __init__(self, rule): if isinstance(self.googlechat_webhook_url, basestring): self.googlechat_webhook_url = [self.googlechat_webhook_url] self.googlechat_format = self.rule.get('googlechat_format', 'basic') - self.googlechat_header_title = self.rule.get('googlechat_header_title', None) + self.googlechat_header_title = self.rule.get('googlechat_header_title', None) self.googlechat_header_subtitle = self.rule.get('googlechat_header_subtitle', None) self.googlechat_header_image = self.rule.get('googlechat_header_image', None) self.googlechat_footer_kibanalink = self.rule.get('googlechat_footer_kibanalink', None) @@ -1472,18 +1472,18 @@ def create_footer(self): footer = None if self.googlechat_footer_kibanalink: footer = {"widgets": [{ - "buttons": [{ - "textButton": { - "text": "VISIT KIBANA", - "onClick": { - "openLink": { - "url": self.googlechat_footer_kibanalink - } + "buttons": [{ + "textButton": { + "text": "VISIT KIBANA", + "onClick": { + "openLink": { + "url": self.googlechat_footer_kibanalink } } - }] - }] - } + } + }] + }] + } return footer def create_card(self, matches): From 4eab5f5f925e5414e545492d8dd47171d1d6e36f Mon Sep 17 00:00:00 2001 From: Alex Makhinov Date: Thu, 13 Sep 2018 20:38:01 +0300 Subject: [PATCH 063/264] Alert hive (#1889) * Add Hive alert --- docs/source/elastalert.rst | 1 + docs/source/ruletypes.rst | 42 +++++++++++++++++++++++++ elastalert/alerts.py | 63 ++++++++++++++++++++++++++++++++++++++ elastalert/config.py | 3 +- requirements.txt | 3 ++ setup.py | 3 ++ 6 files changed, 114 insertions(+), 1 deletion(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index fd2daec8c..b4dcbde44 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -41,6 +41,7 @@ Currently, we have support built in for these alert types: - Telegram - Debug - Stomp +- theHive Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 00b0eeb4b..e164f8013 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1918,3 +1918,45 @@ Example usage:: jira_priority: $priority$ jira_alert_owner: $owner$ + + +theHive +~~~~~~~ + +theHive alert type will send JSON request to theHive (Security Incident Response Platform) with TheHive4py API. Sent request will be stored like Hive Alert with description and observables. + +Required: + +Need to configure connection details for TheHive into the Elastalert rule, example:: + + hive_connection: + hive_host: http://localhost + hive_port: + hive_apikey: + + hive_proxies: + http: '' + https: '' + +Configuration request:: + + hive_alert_config: + title: 'Title' ## This will default to {rule[index]_rule[name]} if not provided + type: 'external' + source: 'elastalert' + description: '{match[field1]} {rule[name]} Sample description' + severity: 2 + tags: ['tag1', 'tag2 {rule[name]}'] + tlp: 3 + status: 'New' + follow: True + +Optional: + +If needed, matched data fields can be mapped to TheHive observable types using python string formatting:: + + hive_observable_data_mapping: + - domain: "{match[field1]}_{rule[name]}" + - domain: "{match[field]}" + - ip: "{match[ip_field]}" + diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 0babda00a..f81c5d5ce 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -20,6 +20,7 @@ import boto3 import requests import stomp +import uuid from exotel import Exotel from jira.client import JIRA from jira.exceptions import JIRAError @@ -27,6 +28,8 @@ from requests.exceptions import RequestException from staticconf.loader import yaml_loader from texttable import Texttable +from thehive4py.api import TheHiveApi +from thehive4py.models import Alert, AlertArtifact from twilio.base.exceptions import TwilioRestException from twilio.rest import Client as TwilioClient from util import EAException @@ -1786,3 +1789,63 @@ def get_info(self): return {'type': 'stride', 'stride_cloud_id': self.stride_cloud_id, 'stride_conversation_id': self.stride_conversation_id} + +class HiveAlerter(Alerter): + """ + Use matched data to create alerts containing observables in an instance of TheHive + """ + + required_options = set(['hive_connection', 'hive_alert_config']) + + def alert(self, matches): + + connection_details = self.rule['hive_connection'] + + api = TheHiveApi( + '{hive_host}:{hive_port}'.format(**connection_details), + connection_details.get('hive_apikey',''), + proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), + cert=connection_details.get('hive_verify', False)) + + for match in matches: + context = {'rule': self.rule, 'match': match} + + artifacts = [] + for mapping in self.rule.get('hive_observable_data_mapping', []): + for observable_type, match_data_key in mapping.iteritems(): + try: + if match_data_key.replace("{match[","").replace("]}","") in context['match']: + artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) + except KeyError: + raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) + + alert_config = { + 'artifacts': artifacts, + 'sourceRef': str(uuid.uuid4())[0:6], + 'title': '{rule[index]}_{rule[name]}'.format(**context) + } + alert_config.update(self.rule.get('hive_alert_config', {})) + + for alert_config_field, alert_config_value in alert_config.iteritems(): + if isinstance(alert_config_value, basestring): + alert_config[alert_config_field] = alert_config_value.format(**context) + elif isinstance(alert_config_value, (list, tuple)): + formatted_list = [] + for element in alert_config_value: + try: + formatted_list.append(element.format(**context)) + except: + formatted_list.append(element) + alert_config[alert_config_field] = formatted_list + + alert = Alert(**alert_config) + response = api.create_alert(alert) + if response.status_code != 201: + raise Exception('alert not successfully created in TheHive\n{}'.format(response.text)) + + def get_info(self): + + return { + 'type': 'hivealerter', + 'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '') + } diff --git a/elastalert/config.py b/elastalert/config.py index c6efb3ad2..cf7ee674d 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -81,7 +81,8 @@ 'gitter': alerts.GitterAlerter, 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, - 'post': alerts.HTTPPostAlerter + 'post': alerts.HTTPPostAlerter, + 'hivealerter': alerts.HiveAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list # For example, jira goes before email so the ticket # will be added to the resulting email. diff --git a/requirements.txt b/requirements.txt index 47676fc9b..4f23f2ec1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -16,3 +16,6 @@ requests>=2.0.0 stomp.py>=4.1.17 texttable>=0.8.8 twilio==6.0.0 +thehive4py>=1.4.4 +python-magic>=0.4.15 +cffi>=1.11.5 diff --git a/setup.py b/setup.py index 5f0d99732..f034d097e 100644 --- a/setup.py +++ b/setup.py @@ -45,5 +45,8 @@ 'stomp.py>=4.1.17', 'texttable>=0.8.8', 'twilio>=6.0.0,<6.1', + 'thehive4py>=1.4.4', + 'python-magic>=0.4.15', + 'cffi>=1.11.5' ] ) From d8977804d5f51a7aa9bc038647b509105e2a6e7b Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 13 Sep 2018 12:03:22 -0700 Subject: [PATCH 064/264] Fixed pep8 issues --- elastalert/alerts.py | 36 +++++++++++++++++++----------------- tox.ini | 2 +- 2 files changed, 20 insertions(+), 18 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 4d07a68e5..58491d17f 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -7,6 +7,7 @@ import subprocess import sys import time +import uuid import warnings from email.mime.text import MIMEText from email.utils import formatdate @@ -20,7 +21,6 @@ import boto3 import requests import stomp -import uuid from exotel import Exotel from jira.client import JIRA from jira.exceptions import JIRAError @@ -29,7 +29,8 @@ from staticconf.loader import yaml_loader from texttable import Texttable from thehive4py.api import TheHiveApi -from thehive4py.models import Alert, AlertArtifact +from thehive4py.models import Alert +from thehive4py.models import AlertArtifact from twilio.base.exceptions import TwilioRestException from twilio.rest import Client as TwilioClient from util import EAException @@ -1574,10 +1575,10 @@ def create_header(self): header = None if self.googlechat_header_title: header = { - "title": self.googlechat_header_title, - "subtitle": self.googlechat_header_subtitle, - "imageUrl": self.googlechat_header_image - } + "title": self.googlechat_header_title, + "subtitle": self.googlechat_header_subtitle, + "imageUrl": self.googlechat_header_image + } return header def create_footer(self): @@ -1590,22 +1591,22 @@ def create_footer(self): "onClick": { "openLink": { "url": self.googlechat_footer_kibanalink - } } } - }] + } }] - } + }] + } return footer def create_card(self, matches): card = {"cards": [{ - "sections": [{ - "widgets": [ - {"textParagraph": {"text": self.create_alert_body(matches).encode('UTF-8')}} - ]} - ]} + "sections": [{ + "widgets": [ + {"textParagraph": {"text": self.create_alert_body(matches).encode('UTF-8')}} ]} + ]} + ]} # Add the optional header header = self.create_header() @@ -1989,6 +1990,7 @@ def get_info(self): 'stride_cloud_id': self.stride_cloud_id, 'stride_conversation_id': self.stride_conversation_id} + class HiveAlerter(Alerter): """ Use matched data to create alerts containing observables in an instance of TheHive @@ -2002,7 +2004,7 @@ def alert(self, matches): api = TheHiveApi( '{hive_host}:{hive_port}'.format(**connection_details), - connection_details.get('hive_apikey',''), + connection_details.get('hive_apikey', ''), proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), cert=connection_details.get('hive_verify', False)) @@ -2013,7 +2015,7 @@ def alert(self, matches): for mapping in self.rule.get('hive_observable_data_mapping', []): for observable_type, match_data_key in mapping.iteritems(): try: - if match_data_key.replace("{match[","").replace("]}","") in context['match']: + if match_data_key.replace("{match[", "").replace("]}", "") in context['match']: artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) @@ -2033,7 +2035,7 @@ def alert(self, matches): for element in alert_config_value: try: formatted_list.append(element.format(**context)) - except: + except KeyError: formatted_list.append(element) alert_config[alert_config_field] = formatted_list diff --git a/tox.ini b/tox.ini index e3bb2b945..b8c80496a 100644 --- a/tox.ini +++ b/tox.ini @@ -25,6 +25,6 @@ norecursedirs = .* virtualenv_run docs build venv env [testenv:docs] deps = {[testenv]deps} - sphinx + sphinx==1.6.6 changedir = docs commands = sphinx-build -b html -d build/doctrees source build/html From e3228f15420a3153eac474a2fb433ba7a8af7c40 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?J=C3=BCrgensen=20Jasper=20VACE?= Date: Fri, 14 Sep 2018 12:03:00 +0200 Subject: [PATCH 065/264] Bug fix and doc update for theHiveAlerter Fixes bug in theHive alerter: https://github.com/Yelp/elastalert/issues/1906 Added 'alert: hivealerter' to documentation and added required and optional settings. --- docs/source/ruletypes.rst | 21 +++++++++++++-------- elastalert/alerts.py | 2 +- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 1faf091c5..0e5204708 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1980,7 +1980,19 @@ theHive alert type will send JSON request to theHive (Security Incident Response Required: -Need to configure connection details for TheHive into the Elastalert rule, example:: +``hive_connection``: The connection details as key:values. Required keys are ``hive_host``, ``hive_port`` and ``hive_apikey``. + +``hive_alert_config``: Configuration options for the alert. + +Optional: + +``hive_proxies``: Proxy configuration. + +``hive_observable_data_mapping``: If needed, matched data fields can be mapped to TheHive observable types using python string formatting. + +Example usage:: + + alert: hivealerter hive_connection: hive_host: http://localhost @@ -1991,8 +2003,6 @@ Need to configure connection details for TheHive into the Elastalert rule, examp http: '' https: '' -Configuration request:: - hive_alert_config: title: 'Title' ## This will default to {rule[index]_rule[name]} if not provided type: 'external' @@ -2004,12 +2014,7 @@ Configuration request:: status: 'New' follow: True -Optional: - -If needed, matched data fields can be mapped to TheHive observable types using python string formatting:: - hive_observable_data_mapping: - domain: "{match[field1]}_{rule[name]}" - domain: "{match[field]}" - ip: "{match[ip_field]}" - diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 58491d17f..4a8e75302 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2035,7 +2035,7 @@ def alert(self, matches): for element in alert_config_value: try: formatted_list.append(element.format(**context)) - except KeyError: + except (AttributeError, KeyError): formatted_list.append(element) alert_config[alert_config_field] = formatted_list From 97727cc373e58068daaa47c0928cfed1465c503f Mon Sep 17 00:00:00 2001 From: John Susek Date: Fri, 14 Sep 2018 15:19:06 -0500 Subject: [PATCH 066/264] When getting count only and using formatted output, include single result --- elastalert/test_rule.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index e05384816..3321d5495 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -117,6 +117,7 @@ def test_file(self, conf, args): self.formatted_output['hits'] = num_hits self.formatted_output['days'] = args.days self.formatted_output['terms'] = terms.keys() + self.formatted_output['result'] = terms else: print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else '')) print("\nAvailable terms in first hit:") From c0d9bc1c6d7d894f4688dec357b1461e4d4114b6 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 17 Sep 2018 12:45:23 -0700 Subject: [PATCH 067/264] Fixed a TypeError that occurs for old versions of Elasticsearch --- elastalert/elastalert.py | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 7de726aaa..b8ae7f21f 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -393,9 +393,13 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): self.total_hits = int(res['hits']['total']) if len(res.get('_shards', {}).get('failures', [])) > 0: - errs = [e['reason']['reason'] for e in res['_shards']['failures'] if 'Failed to parse' in e['reason']['reason']] - if len(errs): - raise ElasticsearchException(errs) + try: + errs = [e['reason']['reason'] for e in res['_shards']['failures'] if 'Failed to parse' in e['reason']['reason']] + if len(errs): + raise ElasticsearchException(errs) + except (TypeError, KeyError): + # Different versions of ES have this formatted in different ways. Fallback to str-ing the whole thing + raise ElasticsearchException(str(res['_shards']['failures'])) logging.debug(str(res)) except ElasticsearchException as e: From 9a4ed90af703943743225c15a69897f9abbaf95f Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 17 Sep 2018 14:27:33 -0700 Subject: [PATCH 068/264] Version 0.1.36 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index f034d097e..3dd56f8bd 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.35', + version='0.1.36', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From 67e563a6681dfd185fa09050f89818f2a0762e3f Mon Sep 17 00:00:00 2001 From: Aidan Rowe Date: Wed, 19 Sep 2018 18:18:04 +1000 Subject: [PATCH 069/264] feat: add support for using fields from alert match in a number of PagerDuty fields --- elastalert/alerts.py | 29 +++++++++++++++++++++++++---- 1 file changed, 25 insertions(+), 4 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 4a8e75302..e3b39b307 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1320,10 +1320,14 @@ def __init__(self, rule): self.pagerduty_api_version = self.rule.get('pagerduty_api_version', 'v1') self.pagerduty_v2_payload_class = self.rule.get('pagerduty_v2_payload_class', '') + self.pagerduty_v2_payload_class_args = self.rule.get('pagerduty_v2_payload_class_args', None) self.pagerduty_v2_payload_component = self.rule.get('pagerduty_v2_payload_component', '') + self.pagerduty_v2_payload_component_args = self.rule.get('pagerduty_v2_payload_component_args', None) self.pagerduty_v2_payload_group = self.rule.get('pagerduty_v2_payload_group', '') + self.pagerduty_v2_payload_group_args = self.rule.get('pagerduty_v2_payload_group_args', None) self.pagerduty_v2_payload_severity = self.rule.get('pagerduty_v2_payload_severity', 'critical') self.pagerduty_v2_payload_source = self.rule.get('pagerduty_v2_payload_source', 'ElastAlert') + self.pagerduty_v2_payload_source_args = self.rule.get('pagerduty_v2_payload_source_args', None) if self.pagerduty_api_version == 'v2': self.url = 'https://events.pagerduty.com/v2/enqueue' @@ -1342,11 +1346,11 @@ def alert(self, matches): 'dedup_key': self.get_incident_key(matches), 'client': self.pagerduty_client_name, 'payload': { - 'class': self.pagerduty_v2_payload_class, - 'component': self.pagerduty_v2_payload_component, - 'group': self.pagerduty_v2_payload_group, + 'class': self.resolve_formatted_key(self.pagerduty_v2_payload_class, self.pagerduty_v2_payload_class_args, matches), + 'component': self.resolve_formatted_key(self.pagerduty_v2_payload_component, self.pagerduty_v2_payload_component_args, matches), + 'group': self.resolve_formatted_key(self.pagerduty_v2_payload_group, self.pagerduty_v2_payload_group_args, matches), 'severity': self.pagerduty_v2_payload_severity, - 'source': self.pagerduty_v2_payload_source, + 'source': self.resolve_formatted_key(self.pagerduty_v2_payload_source, self.pagerduty_v2_payload_source_args, matches), 'summary': self.create_title(matches), 'custom_details': { 'information': body.encode('UTF-8'), @@ -1385,6 +1389,23 @@ def alert(self, matches): elif self.pagerduty_event_type == 'acknowledge': elastalert_logger.info("acknowledge sent to PagerDuty") + def resolve_formatted_key(self, key, args, matches): + if args: + key_values = [lookup_es_key(matches[0], arg) for arg in args] + + # Populate values with rule level properties too + for i in range(len(key_values)): + if key_values[i] is None: + key_value = self.rule.get(args[i]) + if key_value: + key_values[i] = key_value + + missing = self.rule.get('alert_missing_value', '') + key_values = [missing if val is None else val for val in key_values] + return key.format(*key_values) + else: + return key + def get_incident_key(self, matches): if self.pagerduty_incident_key_args: incident_key_values = [lookup_es_key(matches[0], arg) for arg in self.pagerduty_incident_key_args] From a00f1406994a540eaca63f54b2d5476c4e4d524e Mon Sep 17 00:00:00 2001 From: Martijn Rondeel Date: Mon, 24 Sep 2018 12:23:22 +0200 Subject: [PATCH 070/264] Update the ElastAlert Kibana Plugin .gif --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0ac17a8ae..2b09e7d44 100644 --- a/README.md +++ b/README.md @@ -89,7 +89,7 @@ Eg: ``--rule this_rule.yaml`` ## Third Party Tools And Extras ### Kibana plugin -![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/kibana-elastalert-plugin-showcase.gif) +![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif) Available at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin). ### Docker From 0f53444881ea0a09b3789a5fe36d4678417f1141 Mon Sep 17 00:00:00 2001 From: Mike Rohland Date: Mon, 24 Sep 2018 12:28:22 +0200 Subject: [PATCH 071/264] fix dockerfile --- Dockerfile-test | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/Dockerfile-test b/Dockerfile-test index 8005e25f8..761a777c6 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,9 +1,7 @@ FROM ubuntu:latest RUN apt-get update && apt-get upgrade -y -RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox - -RUN easy_install pip +RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox python-pip WORKDIR /home/elastalert From 90874d65ace0bce660f3044fbf76d7a1fed06a40 Mon Sep 17 00:00:00 2001 From: Laurent Butti Date: Mon, 24 Sep 2018 14:48:17 +0200 Subject: [PATCH 072/264] "add_metadata_alert" feature to populate alerts with "description", "owner" and "priority" from rules --- docs/source/elastalert.rst | 2 ++ elastalert/elastalert.py | 6 ++++++ elastalert/ruletypes.py | 1 + 3 files changed, 9 insertions(+) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index e41c60dfa..23fad1075 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -190,6 +190,8 @@ The default value is ``False``. Elasticsearch 2.0 - 2.3 does not support dots in ``string_multi_field_name``: If set, the suffix to use for the subfield for string multi-fields in Elasticsearch. The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticsearch 5. +``add_metadata_alert``: If set, alerts will include metadata described in rules (``description``, ``owner`` and ``priority``); set to ``True`` or ``False``. The default is ``False``. + .. _runningelastalert: Running ElastAlert diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index b8ae7f21f..e2c70b1db 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -153,6 +153,7 @@ def __init__(self, args): self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) self.string_multi_field_name = self.conf.get('string_multi_field_name', False) + self.add_metadata_alert = self.conf.get('add_metadata_alert', False) self.writeback_es = elasticsearch_client(self.conf) self._es_version = None @@ -1461,6 +1462,11 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No 'alert_time': alert_time } + if self.add_metadata_alert: + body['description'] = rule['description'] + body['owner'] = rule['owner'] + body['priority'] = rule['priority'] + match_time = lookup_es_key(match, rule['timestamp_field']) if match_time is not None: body['match_time'] = match_time diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index fe54673e0..2f9e33c1e 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -31,6 +31,7 @@ def __init__(self, rules, args=None): self.matches = [] self.rules = rules self.occurrences = {} + self.rules['description'] = self.rules.get('description', '') self.rules['owner'] = self.rules.get('owner', '') self.rules['priority'] = self.rules.get('priority', '2') From bc45902b967a87b647323acaa6da2a5968aadbc3 Mon Sep 17 00:00:00 2001 From: Laurent Butti Date: Tue, 25 Sep 2018 11:57:45 +0200 Subject: [PATCH 073/264] Add "category" --- docs/source/ruletypes.rst | 7 +++++++ elastalert/elastalert.py | 1 + elastalert/ruletypes.py | 1 + 3 files changed, 9 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 0e5204708..e0afec070 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -84,6 +84,8 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``priority`` (int, default 2) | | +--------------------------------------------------------------+ | +| ``category`` (string, default empty string) | | ++--------------------------------------------------------------+ | | ``scan_entire_timeframe`` (bool, default False) | | +--------------------------------------------------------------+ | | ``import`` (string) | | @@ -406,6 +408,11 @@ priority ``priority``: This value will be used to identify the relative priority of the alert. Optionally, this field can be included in any alert type (e.g. for use in email subject/body text). (Optional, int, default 2) +category +^^^^^^^^ + +``category``: This value will be used to identify the category of the alert. Optionally, this field can be included in any alert type (e.g. for use in email subject/body text). (Optional, string, default empty string) + max_query_size ^^^^^^^^^^^^^^ diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index e2c70b1db..d6c4e0dca 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1463,6 +1463,7 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No } if self.add_metadata_alert: + body['category'] = rule['category'] body['description'] = rule['description'] body['owner'] = rule['owner'] body['priority'] = rule['priority'] diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 2f9e33c1e..73d9ab5f1 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -31,6 +31,7 @@ def __init__(self, rules, args=None): self.matches = [] self.rules = rules self.occurrences = {} + self.rules['category'] = self.rules.get('category', '') self.rules['description'] = self.rules.get('description', '') self.rules['owner'] = self.rules.get('owner', '') self.rules['priority'] = self.rules.get('priority', '2') From 4782e66c5724b44b4300dd86285e72e979a87e69 Mon Sep 17 00:00:00 2001 From: Laurent Butti Date: Tue, 25 Sep 2018 12:06:35 +0200 Subject: [PATCH 074/264] Fix documentation --- docs/source/elastalert.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 23fad1075..81eb56271 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -190,7 +190,7 @@ The default value is ``False``. Elasticsearch 2.0 - 2.3 does not support dots in ``string_multi_field_name``: If set, the suffix to use for the subfield for string multi-fields in Elasticsearch. The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticsearch 5. -``add_metadata_alert``: If set, alerts will include metadata described in rules (``description``, ``owner`` and ``priority``); set to ``True`` or ``False``. The default is ``False``. +``add_metadata_alert``: If set, alerts will include metadata described in rules (``category``, ``description``, ``owner`` and ``priority``); set to ``True`` or ``False``. The default is ``False``. .. _runningelastalert: From d385d663801cd2b5ec68d26a744fa167f1da29ea Mon Sep 17 00:00:00 2001 From: Aidan Rowe Date: Wed, 26 Sep 2018 11:10:47 +1000 Subject: [PATCH 075/264] doc: update PagerDuty alert documentation to add new _arg fields --- docs/source/ruletypes.rst | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 0e5204708..1cf1bb4b3 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1724,14 +1724,22 @@ See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 ``pagerduty_v2_payload_class``: Sets the class of the payload. (the event type in PagerDuty) +``pagerduty_v2_payload_class_args``: If set, and ``pagerduty_v2_payload_class`` is a formattable string, Elastalert will format the class based on the provided array of fields from the rule or match. + ``pagerduty_v2_payload_component``: Sets the component of the payload. (what program/interface/etc the event came from) +``pagerduty_v2_payload_component_args``: If set, and ``pagerduty_v2_payload_component`` is a formattable string, Elastalert will format the component based on the provided array of fields from the rule or match. + ``pagerduty_v2_payload_group``: Sets the logical grouping (e.g. app-stack) +``pagerduty_v2_payload_group_args``: If set, and ``pagerduty_v2_payload_group`` is a formattable string, Elastalert will format the group based on the provided array of fields from the rule or match. + ``pagerduty_v2_payload_severity``: Sets the severity of the page. (defaults to `critical`, valid options: `critical`, `error`, `warning`, `info`) ``pagerduty_v2_payload_source``: Sets the source of the event, preferably the hostname or fqdn. +``pagerduty_v2_payload_source_args``: If set, and ``pagerduty_v2_payload_source`` is a formattable string, Elastalert will format the source based on the provided array of fields from the rule or match. + Exotel ~~~~~~ From 9e5ed8448d4e5640184a56403b774a5a4c06156c Mon Sep 17 00:00:00 2001 From: Muhammad Ahsan Ali Date: Thu, 27 Sep 2018 18:00:48 +0200 Subject: [PATCH 076/264] Add alert routing for opsgenie --- Dockerfile-test | 4 +--- elastalert/config.py | 4 ++++ elastalert/opsgenie.py | 31 ++++++++++++++++++++++++++++--- tests/alerts_test.py | 34 ++++++++++++++++++++++++++++++++++ 4 files changed, 67 insertions(+), 6 deletions(-) diff --git a/Dockerfile-test b/Dockerfile-test index 8005e25f8..761a777c6 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,9 +1,7 @@ FROM ubuntu:latest RUN apt-get update && apt-get upgrade -y -RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox - -RUN easy_install pip +RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox python-pip WORKDIR /home/elastalert diff --git a/elastalert/config.py b/elastalert/config.py index c6efb3ad2..4e5bdf306 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -252,6 +252,10 @@ def _dt_to_ts_with_format(dt): rule.setdefault('hipchat_notify', True) rule.setdefault('hipchat_from', '') rule.setdefault('hipchat_ignore_ssl_errors', False) + + # Set OpsGenie options from global config + rule.setdefault('opsgenie_default_receipients', None) + rule.setdefault('opsgenie_default_teams', None) # Make sure we have required options if required_locals - frozenset(rule.keys()): diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index 45bf27dd1..0580b02c9 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -15,11 +15,14 @@ class OpsGenieAlerter(Alerter): def __init__(self, *args): super(OpsGenieAlerter, self).__init__(*args) - self.account = self.rule.get('opsgenie_account') self.api_key = self.rule.get('opsgenie_key', 'key') + self.default_reciepients = self.rule.get('opsgenie_default_receipients',None) self.recipients = self.rule.get('opsgenie_recipients') + self.recipients_args = self.rule.get('opsgenie_recipients_args') + self.default_teams = self.rule.get('opsgenie_default_teams',None) self.teams = self.rule.get('opsgenie_teams') + self.teams_args = self.rule.get('opsgenie_teams_args') self.tags = self.rule.get('opsgenie_tags', []) + ['ElastAlert', self.rule['name']] self.to_addr = self.rule.get('opsgenie_addr', 'https://api.opsgenie.com/v2/alerts') self.custom_message = self.rule.get('opsgenie_message') @@ -29,6 +32,28 @@ def __init__(self, *args): self.opsgenie_proxy = self.rule.get('opsgenie_proxy', None) self.priority = self.rule.get('opsgenie_priority') + def _parse_responders(self, responders, responder_args, matches, default_responders): + if responder_args: + formated_responders = list() + responders_values = dict( (k,lookup_es_key(matches[0], v)) for k, v in responder_args.iteritems()) + responders_values = dict((k, v) for k, v in responders_values.iteritems() if v) + + for responder in responders: + responder = unicode(responder) + try: + formated_responders.append(responder.format(**responders_values)) + except KeyError as error: + logging.warn("OpsGenieAlerter: Cannot create responder for OpsGenie Alert. Key not foud: %s. "%(error)) + if not formated_responders: + logging.warn("OpsGenieAlerter: no responders can be formed. Trying the default responder ") + if not default_responders: + logging.warn("OpsGenieAlerter: default responder not set. Falling back") + formated_responders = responders + else: + formated_responders = default_responders + responders = formated_responders + return responders + def _fill_responders(self, responders, type_): return [{'id': r, 'type': type_} for r in responders] @@ -44,7 +69,8 @@ def alert(self, matches): self.message = self.create_title(matches) else: self.message = self.custom_message.format(**matches[0]) - + self.recipients = self._parse_responders(self.recipients, self.recipients_args, matches, self.default_reciepients) + self.teams = self._parse_responders(self.teams, self.teams_args, matches, self.default_teams) post = {} post['message'] = self.message if self.account: @@ -129,5 +155,4 @@ def get_info(self): ret['account'] = self.account if self.teams: ret['teams'] = self.teams - return ret diff --git a/tests/alerts_test.py b/tests/alerts_test.py index cf46a4ba2..abd757969 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -420,6 +420,40 @@ def test_opsgenie_frequency(): assert mcal[0][1]['json']['source'] == 'ElastAlert' +def test_opsgenie_alert_routing(): + rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey', + 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts', + 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX':'recipient'}, + 'type': mock_rule(), + 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}], + 'alert': 'opsgenie', + 'opsgenie_teams':['{TEAM_PREFIX}-Team'],'opsgenie_teams_args':{'TEAM_PREFIX':'team'}} + with mock.patch('requests.post') as mock_post: + + alert = OpsGenieAlerter(rule) + alert.alert([{'@timestamp': '2014-10-31T00:00:00','team':"Test",'recipient':"lytics"}]) + + assert alert.get_info()['teams'] == ['Test-Team'] + assert alert.get_info()['recipients'] == ['lytics'] + +def test_opsgenie_default_alert_routing(): + rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey', + 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts', + 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX':'recipient'}, + 'type': mock_rule(), + 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}], + 'alert': 'opsgenie', + 'opsgenie_teams':['{TEAM_PREFIX}-Team'], + 'opsgenie_default_receipients':["devops@test.com"],'opsgenie_default_teams':["Test"] + } + with mock.patch('requests.post') as mock_post: + + alert = OpsGenieAlerter(rule) + alert.alert([{'@timestamp': '2014-10-31T00:00:00','team':"Test"}]) + + assert alert.get_info()['teams'] == ['{TEAM_PREFIX}-Team'] + assert alert.get_info()['recipients'] == ['devops@test.com'] + def test_jira(): description_txt = "Description stuff goes here like a runbook link." rule = { From 401e1f90621650f9e756e9f4c79a1c3c20b69cbf Mon Sep 17 00:00:00 2001 From: Muhammad Ahsan Ali Date: Fri, 28 Sep 2018 16:50:36 +0200 Subject: [PATCH 077/264] Fixing Lint errors and adding example --- elastalert/config.py | 2 +- elastalert/opsgenie.py | 8 ++++---- example_rules/example_opsgenie_frequency.yaml | 14 +++++++++++++ tests/alerts_test.py | 20 ++++++++++--------- 4 files changed, 30 insertions(+), 14 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index 7b31b1cfd..8d60c91d1 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -255,7 +255,7 @@ def _dt_to_ts_with_format(dt): rule.setdefault('hipchat_notify', True) rule.setdefault('hipchat_from', '') rule.setdefault('hipchat_ignore_ssl_errors', False) - + # Set OpsGenie options from global config rule.setdefault('opsgenie_default_receipients', None) rule.setdefault('opsgenie_default_teams', None) diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index 9cc58ba1c..6a6e7fda7 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -17,10 +17,10 @@ def __init__(self, *args): super(OpsGenieAlerter, self).__init__(*args) self.account = self.rule.get('opsgenie_account') self.api_key = self.rule.get('opsgenie_key', 'key') - self.default_reciepients = self.rule.get('opsgenie_default_receipients',None) + self.default_reciepients = self.rule.get('opsgenie_default_receipients', None) self.recipients = self.rule.get('opsgenie_recipients') self.recipients_args = self.rule.get('opsgenie_recipients_args') - self.default_teams = self.rule.get('opsgenie_default_teams',None) + self.default_teams = self.rule.get('opsgenie_default_teams', None) self.teams = self.rule.get('opsgenie_teams') self.teams_args = self.rule.get('opsgenie_teams_args') self.tags = self.rule.get('opsgenie_tags', []) + ['ElastAlert', self.rule['name']] @@ -35,7 +35,7 @@ def __init__(self, *args): def _parse_responders(self, responders, responder_args, matches, default_responders): if responder_args: formated_responders = list() - responders_values = dict( (k,lookup_es_key(matches[0], v)) for k, v in responder_args.iteritems()) + responders_values = dict((k, lookup_es_key(matches[0], v)) for k, v in responder_args.iteritems()) responders_values = dict((k, v) for k, v in responders_values.iteritems() if v) for responder in responders: @@ -43,7 +43,7 @@ def _parse_responders(self, responders, responder_args, matches, default_respond try: formated_responders.append(responder.format(**responders_values)) except KeyError as error: - logging.warn("OpsGenieAlerter: Cannot create responder for OpsGenie Alert. Key not foud: %s. "%(error)) + logging.warn("OpsGenieAlerter: Cannot create responder for OpsGenie Alert. Key not foud: %s. " % (error)) if not formated_responders: logging.warn("OpsGenieAlerter: no responders can be formed. Trying the default responder ") if not default_responders: diff --git a/example_rules/example_opsgenie_frequency.yaml b/example_rules/example_opsgenie_frequency.yaml index f8c835f46..a3febf773 100755 --- a/example_rules/example_opsgenie_frequency.yaml +++ b/example_rules/example_opsgenie_frequency.yaml @@ -21,11 +21,25 @@ opsgenie_key: ogkey #opsgenie_recipients: # - "neh" +# (Optional) +# OpsGenie recipients with args +# opsgenie_recipients: +# - {recipient} +# opsgenie_teams_args: +# team_prefix:'user.email' + # (Optional) # OpsGenie teams to notify #opsgenie_teams: # - "Infrastructure" +# (Optional) +# OpsGenie teams with args +# opsgenie_teams: +# - {team_prefix}-Team +# opsgenie_teams_args: +# team_prefix:'team' + # (Optional) # OpsGenie alert tags opsgenie_tags: diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 2e3875d01..32cc387b7 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -423,37 +423,39 @@ def test_opsgenie_frequency(): def test_opsgenie_alert_routing(): rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey', 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts', - 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX':'recipient'}, + 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX': 'recipient'}, 'type': mock_rule(), 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}], 'alert': 'opsgenie', - 'opsgenie_teams':['{TEAM_PREFIX}-Team'],'opsgenie_teams_args':{'TEAM_PREFIX':'team'}} - with mock.patch('requests.post') as mock_post: + 'opsgenie_teams': ['{TEAM_PREFIX}-Team'], 'opsgenie_teams_args': {'TEAM_PREFIX': 'team'}} + with mock.patch('requests.post'): alert = OpsGenieAlerter(rule) - alert.alert([{'@timestamp': '2014-10-31T00:00:00','team':"Test",'recipient':"lytics"}]) + alert.alert([{'@timestamp': '2014-10-31T00:00:00', 'team': "Test", 'recipient': "lytics"}]) assert alert.get_info()['teams'] == ['Test-Team'] assert alert.get_info()['recipients'] == ['lytics'] + def test_opsgenie_default_alert_routing(): rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey', 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts', - 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX':'recipient'}, + 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX': 'recipient'}, 'type': mock_rule(), 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}], 'alert': 'opsgenie', - 'opsgenie_teams':['{TEAM_PREFIX}-Team'], - 'opsgenie_default_receipients':["devops@test.com"],'opsgenie_default_teams':["Test"] + 'opsgenie_teams': ['{TEAM_PREFIX}-Team'], + 'opsgenie_default_receipients': ["devops@test.com"], 'opsgenie_default_teams': ["Test"] } - with mock.patch('requests.post') as mock_post: + with mock.patch('requests.post'): alert = OpsGenieAlerter(rule) - alert.alert([{'@timestamp': '2014-10-31T00:00:00','team':"Test"}]) + alert.alert([{'@timestamp': '2014-10-31T00:00:00', 'team': "Test"}]) assert alert.get_info()['teams'] == ['{TEAM_PREFIX}-Team'] assert alert.get_info()['recipients'] == ['devops@test.com'] + def test_jira(): description_txt = "Description stuff goes here like a runbook link." rule = { From ccbce2970ead44e25713932435e0705944965103 Mon Sep 17 00:00:00 2001 From: John Susek Date: Fri, 28 Sep 2018 16:52:13 -0500 Subject: [PATCH 078/264] Add slack_title_link options --- docs/source/ruletypes.rst | 1 + elastalert/alerts.py | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 0e5204708..b17295ef7 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1620,6 +1620,7 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_alert_fields``: You can add additional fields to your slack alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`. +``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. Mattermost ~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 4a8e75302..51f43a445 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1111,6 +1111,7 @@ def __init__(self, rule): self.slack_channel_override = self.rule.get('slack_channel_override', '') if isinstance(self.slack_channel_override, basestring): self.slack_channel_override = [self.slack_channel_override] + self.slack_title_link = self.rule.get('slack_title_link', '') self.slack_emoji_override = self.rule.get('slack_emoji_override', ':ghost:') self.slack_icon_url_override = self.rule.get('slack_icon_url_override', '') self.slack_msg_color = self.rule.get('slack_msg_color', 'danger') @@ -1174,6 +1175,9 @@ def alert(self, matches): else: payload['icon_emoji'] = self.slack_emoji_override + if self.slack_title_link != '': + payload['attachments'][0]['title_link'] = self.slack_title_link + for url in self.slack_webhook_url: for channel_override in self.slack_channel_override: try: From 5685a495b3933e5beb0a39ecb1ad023b540e5443 Mon Sep 17 00:00:00 2001 From: Muhammad Ahsan Ali Date: Mon, 1 Oct 2018 12:25:52 +0200 Subject: [PATCH 079/264] Adding docs and fixing example --- docs/source/ruletypes.rst | 2 ++ example_rules/example_opsgenie_frequency.yaml | 2 +- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 0e5204708..ab6b6007b 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1478,8 +1478,10 @@ Optional: ``opsgenie_account``: The OpsGenie account to integrate with. ``opsgenie_recipients``: A list OpsGenie recipients who will be notified by the alert. +``opsgenie_recipients_args``: Map of arguments used to format opsgenie_recipients. ``opsgenie_teams``: A list of OpsGenie teams to notify (useful for schedules with escalation). +``opsgenie_teams_args``: Map of arguments used to format opsgenie_teams (useful for assigning the alerts to teams based on some data) ``opsgenie_tags``: A list of tags for this alert. diff --git a/example_rules/example_opsgenie_frequency.yaml b/example_rules/example_opsgenie_frequency.yaml index a3febf773..9876f9162 100755 --- a/example_rules/example_opsgenie_frequency.yaml +++ b/example_rules/example_opsgenie_frequency.yaml @@ -25,7 +25,7 @@ opsgenie_key: ogkey # OpsGenie recipients with args # opsgenie_recipients: # - {recipient} -# opsgenie_teams_args: +# opsgenie_recipients_args: # team_prefix:'user.email' # (Optional) From f2a56a42fee766d9b530c486d5f59e4f2443f18c Mon Sep 17 00:00:00 2001 From: Muhammad Ahsan Ali Date: Mon, 1 Oct 2018 12:31:27 +0200 Subject: [PATCH 080/264] Adding docs and fixing example --- docs/source/ruletypes.rst | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ab6b6007b..9ebd28cf1 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1479,10 +1479,10 @@ Optional: ``opsgenie_recipients``: A list OpsGenie recipients who will be notified by the alert. ``opsgenie_recipients_args``: Map of arguments used to format opsgenie_recipients. - +``opsgenie_default_recipients``: List of default recipients to notify when the formatting of opsgenie_recipients is unsuccesful. ``opsgenie_teams``: A list of OpsGenie teams to notify (useful for schedules with escalation). ``opsgenie_teams_args``: Map of arguments used to format opsgenie_teams (useful for assigning the alerts to teams based on some data) - +``opsgenie_default_teams``: List of default teams to notify when the formatting of opsgenie_teams is unsuccesful. ``opsgenie_tags``: A list of tags for this alert. ``opsgenie_message``: Set the OpsGenie message to something other than the rule name. The message can be formatted with fields from the first match e.g. "Error occurred for {app_name} at {timestamp}.". From 9f5be416548c519fba4ea0eea7bda602ee3ca581 Mon Sep 17 00:00:00 2001 From: Aidan Rowe Date: Tue, 2 Oct 2018 11:47:28 +1000 Subject: [PATCH 081/264] fix line length exceeding 140 characters --- elastalert/alerts.py | 16 ++++++++++++---- 1 file changed, 12 insertions(+), 4 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index e3b39b307..447d4f77d 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1346,11 +1346,19 @@ def alert(self, matches): 'dedup_key': self.get_incident_key(matches), 'client': self.pagerduty_client_name, 'payload': { - 'class': self.resolve_formatted_key(self.pagerduty_v2_payload_class, self.pagerduty_v2_payload_class_args, matches), - 'component': self.resolve_formatted_key(self.pagerduty_v2_payload_component, self.pagerduty_v2_payload_component_args, matches), - 'group': self.resolve_formatted_key(self.pagerduty_v2_payload_group, self.pagerduty_v2_payload_group_args, matches), + 'class': self.resolve_formatted_key(self.pagerduty_v2_payload_class, + self.pagerduty_v2_payload_class_args, + matches), + 'component': self.resolve_formatted_key(self.pagerduty_v2_payload_component, + self.pagerduty_v2_payload_component_args, + matches), + 'group': self.resolve_formatted_key(self.pagerduty_v2_payload_group, + self.pagerduty_v2_payload_group_args, + matches), 'severity': self.pagerduty_v2_payload_severity, - 'source': self.resolve_formatted_key(self.pagerduty_v2_payload_source, self.pagerduty_v2_payload_source_args, matches), + 'source': self.resolve_formatted_key(self.pagerduty_v2_payload_source, + self.pagerduty_v2_payload_source_args, + matches), 'summary': self.create_title(matches), 'custom_details': { 'information': body.encode('UTF-8'), From 2ed853373730c1d46f1b5dc7c8ef14c0995c5aba Mon Sep 17 00:00:00 2001 From: Suresh Kumar Panneer Selvam Date: Tue, 9 Oct 2018 15:20:54 +0530 Subject: [PATCH 082/264] Fix starttime in subsequent runs when allow_buffer_time_overlap is enabled --- elastalert/elastalert.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index b8ae7f21f..8b13f8b3e 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -717,7 +717,9 @@ def set_starttime(self, rule, endtime): elif 'previous_endtime' in rule: if rule['previous_endtime'] < buffer_delta: rule['starttime'] = rule['previous_endtime'] - self.adjust_start_time_for_overlapping_agg_query(rule) + self.adjust_start_time_for_overlapping_agg_query(rule) + elif rule.get('allow_buffer_time_overlap'): + rule['starttime'] = buffer_delta else: rule['starttime'] = buffer_delta From 75554dc915bf46dfa96993e00e04c560b32daa82 Mon Sep 17 00:00:00 2001 From: Samuele Kaplun Date: Wed, 10 Oct 2018 10:15:08 +0200 Subject: [PATCH 083/264] Timeout support for Slack Signed-off-by: Samuele Kaplun --- docs/source/ruletypes.rst | 8 +++--- elastalert/alerts.py | 4 ++- tests/alerts_test.py | 57 ++++++++++++++++++++++++++++++++++++--- 3 files changed, 61 insertions(+), 8 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 9b2be5acf..bb8e0ee49 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1479,10 +1479,10 @@ Optional: ``opsgenie_recipients``: A list OpsGenie recipients who will be notified by the alert. ``opsgenie_recipients_args``: Map of arguments used to format opsgenie_recipients. -``opsgenie_default_recipients``: List of default recipients to notify when the formatting of opsgenie_recipients is unsuccesful. +``opsgenie_default_recipients``: List of default recipients to notify when the formatting of opsgenie_recipients is unsuccesful. ``opsgenie_teams``: A list of OpsGenie teams to notify (useful for schedules with escalation). ``opsgenie_teams_args``: Map of arguments used to format opsgenie_teams (useful for assigning the alerts to teams based on some data) -``opsgenie_default_teams``: List of default teams to notify when the formatting of opsgenie_teams is unsuccesful. +``opsgenie_default_teams``: List of default teams to notify when the formatting of opsgenie_teams is unsuccesful. ``opsgenie_tags``: A list of tags for this alert. ``opsgenie_message``: Set the OpsGenie message to something other than the rule name. The message can be formatted with fields from the first match e.g. "Error occurred for {app_name} at {timestamp}.". @@ -1624,6 +1624,8 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. +``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. + Mattermost ~~~~~ @@ -1985,7 +1987,7 @@ Required: ``hive_connection``: The connection details as key:values. Required keys are ``hive_host``, ``hive_port`` and ``hive_apikey``. -``hive_alert_config``: Configuration options for the alert. +``hive_alert_config``: Configuration options for the alert. Optional: diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 51f43a445..c80cfda8c 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1119,6 +1119,7 @@ def __init__(self, rule): self.slack_text_string = self.rule.get('slack_text_string', '') self.slack_alert_fields = self.rule.get('slack_alert_fields', '') self.slack_ignore_ssl_errors = self.rule.get('slack_ignore_ssl_errors', False) + self.slack_timeout = self.rule.get('slack_timeout', 10) def format_body(self, body): # https://api.slack.com/docs/formatting @@ -1187,7 +1188,8 @@ def alert(self, matches): response = requests.post( url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, verify=not self.slack_ignore_ssl_errors, - proxies=proxies) + proxies=proxies, + timeout=self.slack_timeout) warnings.resetwarnings() response.raise_for_status() except RequestException as e: diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 32cc387b7..05b2f15e7 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1004,7 +1004,53 @@ def test_slack_uses_custom_title(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=True + verify=True, + timeout=10 + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_slack_uses_custom_timeout(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert_subject': 'Cool subject', + 'alert': [], + 'slack_timeout': 20 + } + load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'somefield': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'channel': '', + 'icon_emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': rule['alert_subject'], + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'text': '', + 'parse': 'none' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=True, + timeout=20 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1046,7 +1092,8 @@ def test_slack_uses_rule_name_when_custom_title_is_not_provided(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=True + verify=True, + timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1089,7 +1136,8 @@ def test_slack_uses_custom_slack_channel(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=True + verify=True, + timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1148,7 +1196,8 @@ def test_slack_uses_list_of_custom_slack_channel(): data=mock.ANY, headers={'content-type': 'application/json'}, proxies=None, - verify=True + verify=True, + timeout=10 ) assert expected_data1 == json.loads(mock_post_request.call_args_list[0][1]['data']) assert expected_data2 == json.loads(mock_post_request.call_args_list[1][1]['data']) From cf7adde838de70a20e4083baf966c2135027e1b0 Mon Sep 17 00:00:00 2001 From: 0xd012 Date: Thu, 18 Oct 2018 11:11:26 +0200 Subject: [PATCH 084/264] Prefix "metric_key" with "metric_" --- elastalert/ruletypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index fe54673e0..de5146dd2 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1031,7 +1031,7 @@ def __init__(self, *args): if 'max_threshold' not in self.rules and 'min_threshold' not in self.rules: raise EAException("MetricAggregationRule must have at least one of either max_threshold or min_threshold") - self.metric_key = self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type'] + self.metric_key = 'metric_' + self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type'] if not self.rules['metric_agg_type'] in self.allowed_aggregations: raise EAException("metric_agg_type must be one of %s" % (str(self.allowed_aggregations))) From d02b47fb7e177845892fd1a4083d156f5c3ba366 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Tamayo Segarra Date: Tue, 30 Oct 2018 23:54:42 +0100 Subject: [PATCH 085/264] Adding option to ignore SSL certificate for Alerta API. Also, Alerta alerter documentation was improved --- docs/source/ruletypes.rst | 53 ++++++++++++++++++++++----------------- elastalert/alerts.py | 45 ++++++++++++++------------------- 2 files changed, 49 insertions(+), 49 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index bb8e0ee49..7c1672b86 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1870,8 +1870,7 @@ Alerta ~~~~~~ Alerta alerter will post an alert in the Alerta server instance through the alert API endpoint. -The default values will work with a local Alerta server installation with authorization disabled. -See http://alerta.readthedocs.io/en/latest/api/alert.html for more details on the Alerta alert json format. +See http://alerta.readthedocs.io/en/latest/api/alert.html for more details on the Alerta JSON format. For Alerta 5.0 @@ -1881,46 +1880,47 @@ Required: Optional: -``alerta_api_key``: This is the api key for alerta server if required. Default behaviour is that no Authorization header sent with the request. +``alerta_api_key``: This is the api key for alerta server, sent in an ``Authorization`` HTTP header. If not defined, no Authorization header is sent. -``alerta_resource``: The resource name of the generated alert. Defaults to "elastalert". Can be a reference to a part of the match. +``alerta_use_qk_as_resource``: If true and query_key is present, this will override ``alerta_resource`` field with the ``query_key value`` (Can be useful if ``query_key`` is a hostname). -``alerta_service``: A list of service tags for the generated alert. Defaults to "elastalert". Can be a reference to a part of the match. +``alerta_use_match_timestamp``: If true, it will use the timestamp of the first match as the ``createTime`` of the alert. otherwise, the current server time is used. -``alerta_severity``: The severity level of the alert. Defaults to "warning". +``alert_missing_value``: Text to replace any match field not found when formating strings. Defaults to ````. -``alerta_origin``: The origin field for the generated alert. Defaults to "elastalert". Can be a reference to a part of the match. +The following options dictate the values of the API JSON payload: -``alerta_environment``: The environment field for the generated alert. Defaults to "Production". Can be a reference to a part of the match. +``alerta_severity``: Defaults to "warning". -``alerta_group``: The group field for the generated alert. No Default. Can be a reference to a part of the match. +``alerta_timeout``: Defaults 84600 (1 Day). -``alerta_timeout``: The time in seconds before this alert will expire (in Alerta). Default 84600 (1 Day). +``alerta_type``: Defaults to "elastalert". -``alerta_correlate``: A list of alerta events that this one correlates with. Default is an empty list. Can make reference to a part of the match to build the event name. +The following options use Python-like string syntax ``{}`` or ``%()s`` to access parts of the match, similar to the CommandAlerter. Ie: "Alert for {clientip}". +If the referenced key is not found in the match, it is replaced by the text indicated by the option ``alert_missing_value``. -``alerta_tags``: A list of alerta tags. Default is an empty list. Can be a reference to a part of the match. +``alerta_resource``: Defaults to "elastalert". -``alerta_use_qk_as_resource``: If true and query_key is present this will override alerta_resource field with the query key value (Can be useful if query_key is a hostname). +``alerta_service``: Defaults to "elastalert". -``alerta_use_match_timestamp``: If true will use the timestamp of the first match as the createTime of the alert, otherwise the current time is used. Default False. +``alerta_origin``: Defaults to "elastalert". -``alerta_event``: Can make reference to parts of the match to build the event name. Defaults to "elastalert". +``alerta_environment``: Defaults to "Production". -``alerta_text``: Python-style string can be used to make reference to parts of the match. Defaults to "elastalert". +``alerta_group``: Defaults to "". -``alerta_type``: Defaults to "elastalert". +``alerta_correlate``: Defaults to an empty list. -``alerta_value``: Can be a reference to a part of the match. No Default. +``alerta_tags``: Defaults to an empty list. -``alerta_attributes_keys``: List of key names for the Alerta Attributes dictionary +``alerta_event``: Defaults to the rule's name. -``alerta_attributes_values``: List of values for the Alerta Attributes dictionary, corresponding in order to the described keys. Can be a reference to a part of the match. +``alerta_text``: Defaults to the rule's text according to its type. -.. info:: +``alerta_value``: Defaults to "". + +The ``attributes`` dictionary is built by joining the lists from ``alerta_attributes_keys`` and ``alerta_attributes_values``, considered in order. - The optional values use Python-like string syntax ``{}`` or ``%()s`` to access parts of the match, similar to the CommandAlerter. Ie: "Alert for {clientip}" - If the referenced value is not found in the match, it is replaced by ```` or the text indicated by the rule in ``alert_missing_value``. Example usage using old-style format:: @@ -1934,6 +1934,13 @@ Example usage using old-style format:: alerta_text: "Probe %(hostname)s is UP at %(logdate)s GMT" alerta_value: "UP" +Example usage using new-style format:: + + alert: + - alerta + alerta_attributes_values: ["{key}", "{logdate}", "{sender_ip}" ] + alerta_text: "Probe {hostname} is UP at {logdate} GMT" + HTTP POST diff --git a/elastalert/alerts.py b/elastalert/alerts.py index c80cfda8c..808718796 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1752,18 +1752,21 @@ class AlertaAlerter(Alerter): def __init__(self, rule): super(AlertaAlerter, self).__init__(rule) + # Setup defaul parameters self.url = self.rule.get('alerta_api_url', None) - - # Fill up default values self.api_key = self.rule.get('alerta_api_key', None) + self.timeout = self.rule.get('alerta_timeout', 86400) + self.use_match_timestamp = self.rule.get('alerta_use_match_timestamp', False) + self.use_qk_as_resource = self.rule.get('alerta_use_qk_as_resource', False) + self.verify_ssl = not self.rule.get('alerta_api_skip_ssl', False) + self.missing_text = self.rule.get('alert_missing_value', '') + + # Fill up default values of the API JSON payload self.severity = self.rule.get('alerta_severity', 'warning') self.resource = self.rule.get('alerta_resource', 'elastalert') self.environment = self.rule.get('alerta_environment', 'Production') self.origin = self.rule.get('alerta_origin', 'elastalert') self.service = self.rule.get('alerta_service', ['elastalert']) - self.timeout = self.rule.get('alerta_timeout', 86400) - self.use_match_timestamp = self.rule.get('alerta_use_match_timestamp', False) - self.use_qk_as_resource = self.rule.get('alerta_use_qk_as_resource', False) self.text = self.rule.get('alerta_text', 'elastalert') self.type = self.rule.get('alerta_type', 'elastalert') self.event = self.rule.get('alerta_event', 'elastalert') @@ -1774,8 +1777,6 @@ def __init__(self, rule): self.attributes_values = self.rule.get('alerta_attributes_values', []) self.value = self.rule.get('alerta_value', '') - self.missing_text = self.rule.get('alert_missing_value', '') - def alert(self, matches): # Override the resource if requested if self.use_qk_as_resource and 'query_key' in self.rule and lookup_es_key(matches[0], self.rule['query_key']): @@ -1788,8 +1789,7 @@ def alert(self, matches): alerta_payload = self.get_json_payload(matches[0]) try: - - response = requests.post(self.url, data=alerta_payload, headers=headers) + response = requests.post(self.url, data=alerta_payload, headers=headers, verify=self.verify_ssl) response.raise_for_status() except RequestException as e: raise EAException("Error posting to Alerta: %s" % e) @@ -1797,7 +1797,7 @@ def alert(self, matches): def create_default_title(self, matches): title = '%s' % (self.rule['name']) - # If the rule has a query_key, add that value plus timestamp to subject + # If the rule has a query_key, add that value if 'query_key' in self.rule: qk = matches[0].get(self.rule['query_key']) if qk: @@ -1817,17 +1817,11 @@ def get_json_payload(self, match): """ - alerta_service = [resolve_string(a_service, match, self.missing_text) for a_service in self.service] - alerta_tags = [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags] - alerta_correlate = [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate] - alerta_attributes_values = [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values] - alerta_text = resolve_string(self.text, match, self.missing_text) - alerta_text = self.rule['type'].get_match_str([match]) if alerta_text == '' else alerta_text - alerta_event = resolve_string(self.event, match, self.missing_text) - alerta_event = self.create_default_title([match]) if alerta_event == '' else alerta_event - - timestamp_field = self.rule.get('timestamp_field', '@timestamp') - match_timestamp = lookup_es_key(match, timestamp_field) + #Using default text and event title if not defined in rule + alerta_text = self.rule['type'].get_match_str([match]) if self.text == '' else resolve_string(self.text, match, self.missing_text) + alerta_event = self.create_default_title([match]) if self.event == '' else resolve_string(self.event, match, self.missing_text) + + match_timestamp = lookup_es_key(match, self.rule.get('timestamp_field', '@timestamp')) if match_timestamp is None: match_timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ") if self.use_match_timestamp: @@ -1847,10 +1841,10 @@ def get_json_payload(self, match): 'event': alerta_event, 'text': alerta_text, 'value': resolve_string(self.value, match, self.missing_text), - 'service': alerta_service, - 'tags': alerta_tags, - 'correlate': alerta_correlate, - 'attributes': dict(zip(self.attributes_keys, alerta_attributes_values)), + 'service': [resolve_string(a_service, match, self.missing_text) for a_service in self.service], + 'tags': [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags], + 'correlate': [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate], + 'attributes': dict(zip(self.attributes_keys, [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values])), 'rawData': self.create_alert_body([match]), } @@ -1860,7 +1854,6 @@ def get_json_payload(self, match): raise Exception("Error building Alerta request: %s" % e) return payload - class HTTPPostAlerter(Alerter): """ Requested elasticsearch indices are sent by HTTP POST. Encoded with JSON. """ From 69a347f97919df2019c47d7d32574e17c92a5113 Mon Sep 17 00:00:00 2001 From: Jose Ignacio Tamayo Segarra Date: Wed, 31 Oct 2018 00:19:46 +0100 Subject: [PATCH 086/264] Adding SSL Verification values to the mock test for Alerta --- tests/alerts_test.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 05b2f15e7..7925ed5d4 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1965,6 +1965,7 @@ def test_alerta_no_auth(ea): 'alerta_api_url': 'http://elastalerthost:8080/api/alert', 'timeframe': datetime.timedelta(hours=1), 'timestamp_field': u'@timestamp', + 'alerta_api_skip_ssl': True, 'alerta_attributes_keys': ["hostname", "TimestampEvent", "senderIP"], 'alerta_attributes_values': ["%(key)s", "%(logdate)s", "%(sender_ip)s"], 'alerta_correlate': ["ProbeUP", "ProbeDOWN"], @@ -2015,7 +2016,8 @@ def test_alerta_no_auth(ea): alert.url, data=mock.ANY, headers={ - 'content-type': 'application/json'} + 'content-type': 'application/json'}, + verify=False ) assert expected_data == json.loads( mock_post_request.call_args_list[0][1]['data']) @@ -2048,6 +2050,7 @@ def test_alerta_auth(ea): mock_post_request.assert_called_once_with( alert.url, data=mock.ANY, + verify=True, headers={ 'content-type': 'application/json', 'Authorization': 'Key {}'.format(rule['alerta_api_key'])}) @@ -2109,6 +2112,7 @@ def test_alerta_new_style(ea): mock_post_request.assert_called_once_with( alert.url, data=mock.ANY, + verify=True, headers={ 'content-type': 'application/json'} ) From 0b4196c8e60c1f0596de7425b340251b8010332b Mon Sep 17 00:00:00 2001 From: Jose Ignacio Tamayo Segarra Date: Wed, 31 Oct 2018 00:26:26 +0100 Subject: [PATCH 087/264] Correcting flake8 spacing errors --- elastalert/alerts.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 808718796..23c113ea4 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1817,7 +1817,7 @@ def get_json_payload(self, match): """ - #Using default text and event title if not defined in rule + # Using default text and event title if not defined in rule alerta_text = self.rule['type'].get_match_str([match]) if self.text == '' else resolve_string(self.text, match, self.missing_text) alerta_event = self.create_default_title([match]) if self.event == '' else resolve_string(self.event, match, self.missing_text) @@ -1844,7 +1844,8 @@ def get_json_payload(self, match): 'service': [resolve_string(a_service, match, self.missing_text) for a_service in self.service], 'tags': [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags], 'correlate': [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate], - 'attributes': dict(zip(self.attributes_keys, [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values])), + 'attributes': dict(zip(self.attributes_keys, + [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values])), 'rawData': self.create_alert_body([match]), } @@ -1854,6 +1855,7 @@ def get_json_payload(self, match): raise Exception("Error building Alerta request: %s" % e) return payload + class HTTPPostAlerter(Alerter): """ Requested elasticsearch indices are sent by HTTP POST. Encoded with JSON. """ From b1924e0f775f62224a023cc0dd2de4516a9f36a9 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 1 Nov 2018 15:13:43 -0700 Subject: [PATCH 088/264] Allow regex in blacklists/whitelist filter enhancements --- elastalert/elastalert.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index b8ae7f21f..09a8e9a62 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -813,6 +813,13 @@ def enhance_filter(self, rule): return filters = rule['filter'] + additional_terms = [] + for term in rule[listname]: + if not term.startswith('/') or not term.endswith('/'): + additional_terms.append(rule['compare_key'] + ':"' + term + '"') + else: + # These are regular expressions and won't work if they are quoted + additional_terms.append(rule['compare_key'] + ':' + term) additional_terms = [(rule['compare_key'] + ':"' + term + '"') for term in rule[listname]] if listname == 'whitelist': query = "NOT " + " AND NOT ".join(additional_terms) From 13d36c17d867ab0252c5fd1f216a10151b74ae36 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 1 Nov 2018 15:27:45 -0700 Subject: [PATCH 089/264] Fixed tests --- tests/rules_test.py | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/tests/rules_test.py b/tests/rules_test.py index e08646a38..1954b5d54 100644 --- a/tests/rules_test.py +++ b/tests/rules_test.py @@ -1138,7 +1138,7 @@ def test_metric_aggregation(): rule = MetricAggregationRule(rules) - assert rule.rules['aggregation_query_element'] == {'cpu_pct_avg': {'avg': {'field': 'cpu_pct'}}} + assert rule.rules['aggregation_query_element'] == {'metric_cpu_pct_avg': {'avg': {'field': 'cpu_pct'}}} assert rule.crossed_thresholds(None) is False assert rule.crossed_thresholds(0.09) is True @@ -1146,17 +1146,17 @@ def test_metric_aggregation(): assert rule.crossed_thresholds(0.79) is False assert rule.crossed_thresholds(0.81) is True - rule.check_matches(datetime.datetime.now(), None, {'cpu_pct_avg': {'value': None}}) - rule.check_matches(datetime.datetime.now(), None, {'cpu_pct_avg': {'value': 0.5}}) + rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': None}}) + rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.5}}) assert len(rule.matches) == 0 - rule.check_matches(datetime.datetime.now(), None, {'cpu_pct_avg': {'value': 0.05}}) - rule.check_matches(datetime.datetime.now(), None, {'cpu_pct_avg': {'value': 0.95}}) + rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.05}}) + rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.95}}) assert len(rule.matches) == 2 rules['query_key'] = 'qk' rule = MetricAggregationRule(rules) - rule.check_matches(datetime.datetime.now(), 'qk_val', {'cpu_pct_avg': {'value': 0.95}}) + rule.check_matches(datetime.datetime.now(), 'qk_val', {'metric_cpu_pct_avg': {'value': 0.95}}) assert rule.matches[0]['qk'] == 'qk_val' @@ -1170,9 +1170,9 @@ def test_metric_aggregation_complex_query_key(): 'max_threshold': 0.8} query = {"bucket_aggs": {"buckets": [ - {"cpu_pct_avg": {"value": 0.91}, "key": "sub_qk_val1"}, - {"cpu_pct_avg": {"value": 0.95}, "key": "sub_qk_val2"}, - {"cpu_pct_avg": {"value": 0.89}, "key": "sub_qk_val3"}] + {"metric_cpu_pct_avg": {"value": 0.91}, "key": "sub_qk_val1"}, + {"metric_cpu_pct_avg": {"value": 0.95}, "key": "sub_qk_val2"}, + {"metric_cpu_pct_avg": {"value": 0.89}, "key": "sub_qk_val3"}] }, "key": "qk_val"} rule = MetricAggregationRule(rules) From a433335dd2cea4bb7a001523b3f99264d6a1333c Mon Sep 17 00:00:00 2001 From: John Susek Date: Sun, 4 Nov 2018 15:35:51 -0600 Subject: [PATCH 090/264] Add option to gracefully fail loading of configs. --- docs/source/elastalert.rst | 2 ++ elastalert/config.py | 12 +++++++++++- elastalert/elastalert.py | 6 ++++++ 3 files changed, 19 insertions(+), 1 deletion(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index e41c60dfa..7d89735ae 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -190,6 +190,8 @@ The default value is ``False``. Elasticsearch 2.0 - 2.3 does not support dots in ``string_multi_field_name``: If set, the suffix to use for the subfield for string multi-fields in Elasticsearch. The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticsearch 5. +``skip_invalid``: If ``True``, skip invalid files instead of exiting. + .. _runningelastalert: Running ElastAlert diff --git a/elastalert/config.py b/elastalert/config.py index 8d60c91d1..214b70c52 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -116,7 +116,13 @@ def load_configuration(filename, conf, args=None): :param conf: The global configuration dictionary, used for populating defaults. :return: The rule configuration, a dictionary. """ - rule = load_rule_yaml(filename) + try: + rule = load_rule_yaml(filename) + except Exception as e: + if (conf.get('skip_invalid') == True): + return False + else: + raise e load_options(rule, conf, filename, args) load_modules(rule, args) return rule @@ -483,6 +489,10 @@ def load_rules(args): for rule_file in rule_files: try: rule = load_configuration(rule_file, conf, args) + # A rule failed to load, don't try to process it + if (rule == False): + logging.error('Invalid rule file skipped: %s' % rule_file) + continue # By setting "is_enabled: False" in rule file, a rule is easily disabled if 'is_enabled' in rule and not rule['is_enabled']: continue diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 8b13f8b3e..edbd82450 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1042,6 +1042,9 @@ def load_rule_changes(self): # Rule file was changed, reload rule try: new_rule = load_configuration(rule_file, self.conf) + if (new_rule == False): + logging.error('Invalid rule file skipped: %s' % rule_file) + continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: elastalert_logger.info('Rule file %s is now disabled.' % (rule_file)) # Remove this rule if it's been disabled @@ -1080,6 +1083,9 @@ def load_rule_changes(self): for rule_file in set(new_rule_hashes.keys()) - set(self.rule_hashes.keys()): try: new_rule = load_configuration(rule_file, self.conf) + if (new_rule == False): + logging.error('Invalid rule file skipped: %s' % rule_file) + continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: continue if new_rule['name'] in [rule['name'] for rule in self.rules]: From 0d8ecbfcf0dbd51ee627b6b49045b4de6d34d681 Mon Sep 17 00:00:00 2001 From: John Susek Date: Sun, 4 Nov 2018 15:44:55 -0600 Subject: [PATCH 091/264] Fix CI fails --- elastalert/config.py | 4 ++-- elastalert/elastalert.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index 214b70c52..8484da873 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -119,7 +119,7 @@ def load_configuration(filename, conf, args=None): try: rule = load_rule_yaml(filename) except Exception as e: - if (conf.get('skip_invalid') == True): + if (conf.get('skip_invalid')): return False else: raise e @@ -490,7 +490,7 @@ def load_rules(args): try: rule = load_configuration(rule_file, conf, args) # A rule failed to load, don't try to process it - if (rule == False): + if (not rule): logging.error('Invalid rule file skipped: %s' % rule_file) continue # By setting "is_enabled: False" in rule file, a rule is easily disabled diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index edbd82450..0b1296275 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1042,7 +1042,7 @@ def load_rule_changes(self): # Rule file was changed, reload rule try: new_rule = load_configuration(rule_file, self.conf) - if (new_rule == False): + if (not new_rule): logging.error('Invalid rule file skipped: %s' % rule_file) continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: @@ -1083,7 +1083,7 @@ def load_rule_changes(self): for rule_file in set(new_rule_hashes.keys()) - set(self.rule_hashes.keys()): try: new_rule = load_configuration(rule_file, self.conf) - if (new_rule == False): + if (not new_rule): logging.error('Invalid rule file skipped: %s' % rule_file) continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: From cc8c1cde1a15dfb678546d032e6b24b4e06046f2 Mon Sep 17 00:00:00 2001 From: John Susek Date: Sun, 4 Nov 2018 20:29:17 -0600 Subject: [PATCH 092/264] Add exception to output --- elastalert/config.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/config.py b/elastalert/config.py index 8484da873..d1c27f44a 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -120,6 +120,7 @@ def load_configuration(filename, conf, args=None): rule = load_rule_yaml(filename) except Exception as e: if (conf.get('skip_invalid')): + logging.error(e) return False else: raise e From d37b4938425b77ff7c360f93d56c25c48928a0d9 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 6 Nov 2018 16:42:20 -0800 Subject: [PATCH 093/264] Fixed a bug with additional_terms and regex --- elastalert/elastalert.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 09a8e9a62..284f7b205 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -820,7 +820,6 @@ def enhance_filter(self, rule): else: # These are regular expressions and won't work if they are quoted additional_terms.append(rule['compare_key'] + ':' + term) - additional_terms = [(rule['compare_key'] + ':"' + term + '"') for term in rule[listname]] if listname == 'whitelist': query = "NOT " + " AND NOT ".join(additional_terms) else: From 43927302b0d37ef05540328a4c57e930dff9c0af Mon Sep 17 00:00:00 2001 From: aedades Date: Wed, 7 Nov 2018 11:55:58 -0800 Subject: [PATCH 094/264] Fix _ref and _cur in comment --- example_rules/example_spike.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/example_rules/example_spike.yaml b/example_rules/example_spike.yaml index 48e1f029d..cb7064c2e 100755 --- a/example_rules/example_spike.yaml +++ b/example_rules/example_spike.yaml @@ -31,7 +31,7 @@ index: logstash-* # (Required one of _cur or _ref, spike specific) # The minimum number of events that will trigger an alert # For example, if there are only 2 events between 12:00 and 2:00, and 20 between 2:00 and 4:00 -# _cur is 2 and _ref is 20, and the alert WILL fire because 20 is greater than threshold_cur +# _ref is 2 and _cur is 20, and the alert WILL fire because 20 is greater than threshold_cur and (_ref * spike_height) threshold_cur: 5 #threshold_ref: 5 From 423ce59b75f94e9aad415982a5dba9164e429876 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 8 Nov 2018 13:06:56 -0800 Subject: [PATCH 095/264] Version 0.1.37 --- changelog.md | 11 +++++++++++ setup.py | 2 +- 2 files changed, 12 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 176eda54a..348e70f74 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,16 @@ # Change Log +# v0.1.36 + +### Added +- Added a prefix "metric_" to the key used for metric aggregations to avoid possible conflicts +- Added option to skip Alerta certificate validation +- Added option to gracefully fail when loading rules + +### Fixed +- No longer puts quotes around regex terms in blacklists or whitelists +- Fixed a typo in the documentation for spike rule + # v0.1.35 ### Fixed diff --git a/setup.py b/setup.py index 3dd56f8bd..865d7974f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.36', + version='0.1.37', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From df481d045c7c0efb9a46c268556ac6300564c19d Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 9 Nov 2018 14:29:46 -0800 Subject: [PATCH 096/264] Fixed and merge mater with threading and scheduling --- elastalert/config.py | 3 + elastalert/create_index.py | 2 + elastalert/elastalert.py | 229 ++++++++++++++++++++++++------------- elastalert/enhancements.py | 6 + elastalert/util.py | 13 ++- requirements-dev.txt | 1 + requirements.txt | 7 +- setup.py | 1 + tests/base_test.py | 65 +++++------ tests/conftest.py | 15 ++- 10 files changed, 220 insertions(+), 122 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index d1c27f44a..8a9f73995 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -100,6 +100,7 @@ def get_module(module_name): """ Loads a module and returns a specific object. module_name should 'module.file.object'. Returns object or raises EAException on error. """ + sys.path.append(os.getcwd()) try: module_path, module_class = module_name.rsplit('.', 1) base_module = __import__(module_path, globals(), locals(), [module_class]) @@ -195,6 +196,8 @@ def load_options(rule, conf, filename, args=None): rule['query_delay'] = datetime.timedelta(**rule['query_delay']) if 'buffer_time' in rule: rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) + if 'run_every' in rule: + rule['run_every'] = datetime.timedelta(**rule['run_every']) if 'bucket_interval' in rule: rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) if 'exponential_realert' in rule: diff --git a/elastalert/create_index.py b/elastalert/create_index.py index b12ee7e5e..953c3a1ee 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -57,6 +57,8 @@ def main(): filename = 'config.yaml' elif os.path.isfile(args.config): filename = args.config + elif os.path.isfile('../config.yaml'): + filename = '../config.yaml' else: filename = '' diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 29cc87568..e889161cd 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -5,8 +5,10 @@ import json import logging import os +import random import signal import sys +import threading import time import timeit import traceback @@ -17,8 +19,10 @@ import dateutil.tz import kibana +import pytz import yaml from alerts import DebugAlerter +from apscheduler.schedulers.background import BackgroundScheduler from config import get_rule_hashes from config import load_configuration from config import load_rules @@ -63,6 +67,8 @@ class ElastAlerter(): should not be passed directly from a configuration file, but must be populated by config.py:load_rules instead. """ + thread_data = threading.local() + def parse_args(self, args): parser = argparse.ArgumentParser() parser.add_argument( @@ -128,6 +134,7 @@ def __init__(self, args): tracer.addHandler(logging.FileHandler(self.args.es_debug_trace)) self.conf = load_rules(self.args) + print len(self.conf['rules']), 'rules loaded' self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] self.rules = self.conf['rules'] @@ -140,18 +147,15 @@ def __init__(self, args): self.from_addr = self.conf.get('from_addr', 'ElastAlert') self.smtp_host = self.conf.get('smtp_host', 'localhost') self.max_aggregation = self.conf.get('max_aggregation', 10000) - self.alerts_sent = 0 - self.cumulative_hits = 0 - self.num_hits = 0 - self.num_dupes = 0 - self.current_es = None - self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} self.rule_hashes = get_rule_hashes(self.conf, self.args.rule) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.scheduler = BackgroundScheduler() self.string_multi_field_name = self.conf.get('string_multi_field_name', False) self.add_metadata_alert = self.conf.get('add_metadata_alert', False) @@ -299,7 +303,8 @@ def get_index_start(self, index, timestamp_field='@timestamp'): """ query = {'sort': {timestamp_field: {'order': 'asc'}}} try: - res = self.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field], ignore_unavailable=True) + res = self.thread_data.current_es.search(index=index, size=1, body=query, + _source_include=[timestamp_field], ignore_unavailable=True) except ElasticsearchException as e: self.handle_error("Elasticsearch query error: %s" % (e), {'index': index, 'query': query}) return '1969-12-30T00:00:00Z' @@ -381,9 +386,9 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): try: if scroll: - res = self.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive) + res = self.thread_data.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive) else: - res = self.current_es.search( + res = self.thread_data.current_es.search( scroll=scroll_keepalive, index=index, size=rule.get('max_query_size', self.max_query_size), @@ -391,7 +396,7 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): ignore_unavailable=True, **extra_args ) - self.total_hits = int(res['hits']['total']) + self.thread_data.total_hits = int(res['hits']['total']) if len(res.get('_shards', {}).get('failures', [])) > 0: try: @@ -411,16 +416,16 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): self.handle_error('Error running query: %s' % (e), {'rule': rule['name'], 'query': query}) return None hits = res['hits']['hits'] - self.num_hits += len(hits) + self.thread_data.num_hits += len(hits) lt = rule.get('use_local_time') status_log = "Queried rule %s from %s to %s: %s / %s hits" % ( rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), - self.num_hits, + self.thread_data.num_hits, len(hits) ) - if self.total_hits > rule.get('max_query_size', self.max_query_size): + if self.thread_data.total_hits > rule.get('max_query_size', self.max_query_size): elastalert_logger.info("%s (scrolling..)" % status_log) rule['scroll_id'] = res['_scroll_id'] else: @@ -454,7 +459,7 @@ def get_hits_count(self, rule, starttime, endtime, index): ) try: - res = self.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True) + res = self.thread_data.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -463,7 +468,7 @@ def get_hits_count(self, rule, starttime, endtime, index): self.handle_error('Error running count query: %s' % (e), {'rule': rule['name'], 'query': query}) return None - self.num_hits += res['count'] + self.thread_data.num_hits += res['count'] lt = rule.get('use_local_time') elastalert_logger.info( "Queried rule %s from %s to %s: %s hits" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), res['count']) @@ -509,7 +514,7 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non try: if not rule['five']: - res = self.current_es.search( + res = self.thread_data.current_es.search( index=index, doc_type=rule['doc_type'], body=query, @@ -517,7 +522,8 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule['doc_type'], body=query, size=0, ignore_unavailable=True) + res = self.thread_data.current_es.search(index=index, doc_type=rule['doc_type'], + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -532,7 +538,7 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non buckets = res['aggregations']['filtered']['counts']['buckets'] else: buckets = res['aggregations']['counts']['buckets'] - self.num_hits += len(buckets) + self.thread_data.num_hits += len(buckets) lt = rule.get('use_local_time') elastalert_logger.info( 'Queried rule %s from %s to %s: %s buckets' % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), len(buckets)) @@ -555,7 +561,7 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ query = self.get_aggregation_query(base_query, rule, query_key, term_size, rule['timestamp_field']) try: if not rule['five']: - res = self.current_es.search( + res = self.thread_data.current_es.search( index=index, doc_type=rule.get('doc_type'), body=query, @@ -563,7 +569,8 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule.get('doc_type'), body=query, size=0, ignore_unavailable=True) + res = self.thread_data.current_es.search(index=index, doc_type=rule.get('doc_type'), + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: if len(str(e)) > 1024: e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024) @@ -575,7 +582,7 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ payload = res['aggregations']['filtered'] else: payload = res['aggregations'] - self.num_hits += res['hits']['total'] + self.thread_data.num_hits += res['hits']['total'] return {endtime: payload} def remove_duplicate_events(self, data, rule): @@ -629,7 +636,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): if data: old_len = len(data) data = self.remove_duplicate_events(data, rule) - self.num_dupes += old_len - len(data) + self.thread_data.num_dupes += old_len - len(data) # There was an exception while querying if data is None: @@ -645,7 +652,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): rule_inst.add_data(data) try: - if rule.get('scroll_id') and self.num_hits < self.total_hits: + if rule.get('scroll_id') and self.thread_data.num_hits < self.thread_data.total_hits: self.run_query(rule, start, end, scroll=True) except RuntimeError: # It's possible to scroll far enough to hit max recursive depth @@ -689,7 +696,6 @@ def get_starttime(self, rule): def set_starttime(self, rule, endtime): """ Given a rule and an endtime, sets the appropriate starttime for it. """ - # This means we are starting fresh if 'starttime' not in rule: if not rule.get('scan_entire_timeframe'): @@ -844,8 +850,7 @@ def run_rule(self, rule, endtime, starttime=None): """ run_start = time.time() - self.current_es = elasticsearch_client(rule) - self.current_es_addr = (rule['es_host'], rule['es_port']) + self.thread_data.current_es = elasticsearch_client(rule) # If there are pending aggregate matches, try processing them for x in range(len(rule['agg_matches'])): @@ -866,9 +871,9 @@ def run_rule(self, rule, endtime, starttime=None): return 0 # Run the rule. If querying over a large time period, split it up into segments - self.num_hits = 0 - self.num_dupes = 0 - self.cumulative_hits = 0 + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.thread_data.cumulative_hits = 0 segment_size = self.get_segment_size(rule) tmp_endtime = rule['starttime'] @@ -877,15 +882,15 @@ def run_rule(self, rule, endtime, starttime=None): tmp_endtime = tmp_endtime + segment_size if not self.run_query(rule, rule['starttime'], tmp_endtime): return 0 - self.cumulative_hits += self.num_hits - self.num_hits = 0 + self.thread_data.cumulative_hits += self.thread_data.num_hits + self.thread_data.num_hits = 0 rule['starttime'] = tmp_endtime rule['type'].garbage_collect(tmp_endtime) if rule.get('aggregation_query_element'): if endtime - tmp_endtime == segment_size: self.run_query(rule, tmp_endtime, endtime) - self.cumulative_hits += self.num_hits + self.thread_data.cumulative_hits += self.thread_data.num_hits elif total_seconds(rule['original_starttime'] - tmp_endtime) == 0: rule['starttime'] = rule['original_starttime'] return 0 @@ -894,14 +899,14 @@ def run_rule(self, rule, endtime, starttime=None): else: if not self.run_query(rule, rule['starttime'], endtime): return 0 - self.cumulative_hits += self.num_hits + self.thread_data.cumulative_hits += self.thread_data.num_hits rule['type'].garbage_collect(endtime) # Process any new matches num_matches = len(rule['type'].matches) while rule['type'].matches: match = rule['type'].matches.pop(0) - match['num_hits'] = self.cumulative_hits + match['num_hits'] = self.thread_data.cumulative_hits match['num_matches'] = num_matches # If realert is set, silence the rule for that duration @@ -947,7 +952,7 @@ def run_rule(self, rule, endtime, starttime=None): 'endtime': endtime, 'starttime': rule['original_starttime'], 'matches': num_matches, - 'hits': max(self.num_hits, self.cumulative_hits), + 'hits': max(self.thread_data.num_hits, self.thread_data.cumulative_hits), '@timestamp': ts_now(), 'time_taken': time_taken} self.writeback('elastalert_status', body) @@ -956,6 +961,9 @@ def run_rule(self, rule, endtime, starttime=None): def init_rule(self, new_rule, new=True): ''' Copies some necessary non-config state from an exiting rule to a new rule. ''' + if not new: + self.scheduler.remove_job(job_id=new_rule['name']) + try: self.modify_rule_for_ES5(new_rule) except TransportError as e: @@ -990,7 +998,9 @@ def init_rule(self, new_rule, new=True): blank_rule = {'agg_matches': [], 'aggregate_alert_time': {}, 'current_aggregate_id': {}, - 'processed_hits': {}} + 'processed_hits': {}, + 'run_every': self.run_every, + 'has_run_once': False} rule = blank_rule # Set rule to either a blank template or existing rule with same name @@ -1006,12 +1016,22 @@ def init_rule(self, new_rule, new=True): 'aggregate_alert_time', 'processed_hits', 'starttime', - 'minimum_starttime'] + 'minimum_starttime', + 'has_run_once', + 'run_every'] for prop in copy_properties: if prop not in rule: continue new_rule[prop] = rule[prop] + job = self.scheduler.add_job(self.handle_rule_execution, 'interval', + args=[new_rule], + seconds=new_rule['run_every'].total_seconds(), + id=new_rule['name'], + max_instances=1, + jitter=5) + job.modify(next_run_time=datetime.datetime.now() + datetime.timedelta(seconds=random.randint(0, 15))) + return new_rule @staticmethod @@ -1118,14 +1138,20 @@ def start(self): except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (self.starttime)) exit(1) + + for rule in self.rules: + rule['initial_starttime'] = self.starttime self.wait_until_responsive(timeout=self.args.timeout) self.running = True elastalert_logger.info("Starting up") + self.scheduler.add_job(self.handle_pending_alerts, 'interval', + seconds=self.run_every.total_seconds(), id='_internal_handle_pending_alerts') + self.scheduler.add_job(self.handle_config_change, 'interval', + seconds=self.run_every.total_seconds(), id='_internal_handle_config_change') + self.scheduler.start() while self.running: next_run = datetime.datetime.utcnow() + self.run_every - self.run_all_rules() - # Quit after end_time has been reached if self.args.end: endtime = ts_to_dt(self.args.end) @@ -1176,53 +1202,95 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): def run_all_rules(self): """ Run each rule one time """ + self.handle_pending_alerts() + + for rule in self.rules: + self.handle_rule_execution(rule) + + self.handle_config_change() + + def handle_pending_alerts(self): + self.thread_data.alerts_sent = 0 self.send_pending_alerts() + elastalert_logger.info("Background alerts thread %s pending alerts sent at %s" % (self.thread_data.alerts_sent, + pretty_ts(ts_now()))) - next_run = datetime.datetime.utcnow() + self.run_every + def handle_config_change(self): + if not self.args.pin_rules: + self.load_rule_changes() + elastalert_logger.info("Background configuration change check run at %s" % (pretty_ts(ts_now()))) + + def handle_rule_execution(self, rule): + self.thread_data.alerts_sent = 0 + next_run = datetime.datetime.utcnow() + rule['run_every'] + # Set endtime based on the rule's delay + delay = rule.get('query_delay') + if hasattr(self.args, 'end') and self.args.end: + endtime = ts_to_dt(self.args.end) + elif delay: + endtime = ts_now() - delay + else: + endtime = ts_now() + + # Apply rules based on execution time limits + if rule.get('limit_execution'): + rule['next_starttime'] = None + rule['next_min_starttime'] = None + exec_next = croniter(rule['limit_execution']).next() + endtime_epoch = dt_to_unix(endtime) + # If the estimated next endtime (end + run_every) isn't at least a minute past the next exec time + # That means that we need to pause execution after this run + if endtime_epoch + rule['run_every'].total_seconds() < exec_next - 59: + # apscheduler requires pytz tzinfos, so don't use unix_to_dt here! + rule['next_starttime'] = datetime.datetime.utcfromtimestamp(exec_next).replace(tzinfo=pytz.utc) + if rule.get('limit_execution_coverage'): + rule['next_min_starttime'] = rule['next_starttime'] + if not rule['has_run_once']: + self.reset_rule_schedule(rule) + return - for rule in self.rules: - # Set endtime based on the rule's delay - delay = rule.get('query_delay') - if hasattr(self.args, 'end') and self.args.end: - endtime = ts_to_dt(self.args.end) - elif delay: - endtime = ts_now() - delay - else: - endtime = ts_now() + rule['has_run_once'] = True + try: + num_matches = self.run_rule(rule, endtime, rule.get('initial_starttime')) + except EAException as e: + self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']}) + except Exception as e: + self.handle_uncaught_exception(e, rule) + else: + old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time')) + elastalert_logger.info("Ran %s from %s to %s: %s query hits (%s already seen), %s matches," + " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')), + self.thread_data.num_hits, self.thread_data.num_dupes, num_matches, + self.thread_data.alerts_sent)) + self.thread_data.alerts_sent = 0 - try: - num_matches = self.run_rule(rule, endtime, self.starttime) - except EAException as e: - self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']}) - except Exception as e: - self.handle_uncaught_exception(e, rule) - else: - old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time')) - total_hits = max(self.num_hits, self.cumulative_hits) - elastalert_logger.info("Ran %s from %s to %s: %s query hits (%s already seen), %s matches," - " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')), - total_hits, self.num_dupes, num_matches, self.alerts_sent)) - self.alerts_sent = 0 - - if next_run < datetime.datetime.utcnow(): - # We were processing for longer than our refresh interval - # This can happen if --start was specified with a large time period - # or if we are running too slow to process events in real time. - logging.warning( - "Querying from %s to %s took longer than %s!" % ( - old_starttime, - pretty_ts(endtime, rule.get('use_local_time')), - self.run_every - ) + if next_run < datetime.datetime.utcnow(): + # We were processing for longer than our refresh interval + # This can happen if --start was specified with a large time period + # or if we are running too slow to process events in real time. + logging.warning( + "Querying from %s to %s took longer than %s!" % ( + old_starttime, + pretty_ts(endtime, rule.get('use_local_time')), + self.run_every ) + ) - self.remove_old_events(rule) + rule['initial_starttime'] = None - # Only force starttime once - self.starttime = None + self.remove_old_events(rule) - if not self.args.pin_rules: - self.load_rule_changes() + self.reset_rule_schedule(rule) + + def reset_rule_schedule(self, rule): + # We hit the end of a execution schedule, pause ourselves until next run + if rule.get('limit_execution') and rule['next_starttime']: + self.scheduler.modify_job(job_id=rule['name'], next_run_time=rule['next_starttime']) + # If we are preventing covering non-scheduled time periods, reset min_starttime and previous_endtime + if rule['next_min_starttime']: + rule['minimum_starttime'] = rule['next_min_starttime'] + rule['previous_endtime'] = rule['next_min_starttime'] + elastalert_logger.info('Pausing %s until next run at %s' % (rule['name'], pretty_ts(rule['next_starttime']))) def stop(self): """ Stop an ElastAlert runner that's been started """ @@ -1453,7 +1521,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): self.handle_error('Error while running alert %s: %s' % (alert.get_info()['type'], e), {'rule': rule['name']}) alert_exception = str(e) else: - self.alerts_sent += 1 + self.thread_data.alerts_sent += 1 alert_sent = True # Write the alert(s) to ES @@ -1834,6 +1902,7 @@ def handle_uncaught_exception(self, exception, rule): if self.disable_rules_on_error: self.rules = [running_rule for running_rule in self.rules if running_rule['name'] != rule['name']] self.disabled_rules.append(rule) + self.scheduler.pause_job(job_id=rule['name']) elastalert_logger.info('Rule %s disabled', rule['name']) if self.notify_email: self.send_notification_email(exception=exception, rule=rule) @@ -1892,7 +1961,7 @@ def get_top_counts(self, rule, starttime, endtime, keys, number=None, qk=None): buckets = hits_terms.values()[0] # get_hits_terms adds to num_hits, but we don't want to count these - self.num_hits -= len(buckets) + self.thread_data.num_hits -= len(buckets) terms = {} for bucket in buckets: terms[bucket['key']] = bucket['doc_count'] diff --git a/elastalert/enhancements.py b/elastalert/enhancements.py index d6c902514..2744e35c8 100644 --- a/elastalert/enhancements.py +++ b/elastalert/enhancements.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from util import pretty_ts class BaseEnhancement(object): @@ -14,6 +15,11 @@ def process(self, match): raise NotImplementedError() +class TimeEnhancement(BaseEnhancement): + def process(self, match): + match['@timestamp'] = pretty_ts(match['@timestamp']) + + class DropMatchException(Exception): """ ElastAlert will drop a match if this exception type is raised by an enhancement """ pass diff --git a/elastalert/util.py b/elastalert/util.py index 33f0b4e71..29cf24fbe 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -5,7 +5,7 @@ import os import dateutil.parser -import dateutil.tz +import pytz from auth import Auth from elasticsearch import RequestsHttpConnection from elasticsearch.client import Elasticsearch @@ -112,7 +112,7 @@ def ts_to_dt(timestamp): dt = dateutil.parser.parse(timestamp) # Implicitly convert local timestamps to UTC if dt.tzinfo is None: - dt = dt.replace(tzinfo=dateutil.tz.tzutc()) + dt = dt.replace(tzinfo=pytz.utc) return dt @@ -365,6 +365,15 @@ def build_es_conn_config(conf): return parsed_conf +def pytzfy(dt): + # apscheduler requires pytz timezone objects + # This function will replace a dateutil.tz one with a pytz one + if dt.tzinfo is not None: + new_tz = pytz.timezone(dt.tzinfo.tzname('Y is this even required??')) + return dt.replace(tzinfo=new_tz) + return dt + + def parse_duration(value): """Convert ``unit=num`` spec into a ``timedelta`` object.""" unit, num = value.split('=') diff --git a/requirements-dev.txt b/requirements-dev.txt index 36daa0ebd..1cb67cb8e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,3 +1,4 @@ +-r requirements.txt coverage flake8 pre-commit diff --git a/requirements.txt b/requirements.txt index 4f23f2ec1..418f92869 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,6 +1,8 @@ +apscheduler>=3.3.0 aws-requests-auth>=0.3.0 blist>=1.3.6 boto3>=1.4.4 +cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 elasticsearch @@ -11,11 +13,10 @@ jsonschema>=2.6.0 mock>=2.0.0 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 +python-magic>=0.4.15 PyYAML>=3.12 requests>=2.0.0 stomp.py>=4.1.17 texttable>=0.8.8 -twilio==6.0.0 thehive4py>=1.4.4 -python-magic>=0.4.15 -cffi>=1.11.5 +twilio==6.0.0 diff --git a/setup.py b/setup.py index 865d7974f..b2528e3b4 100644 --- a/setup.py +++ b/setup.py @@ -27,6 +27,7 @@ packages=find_packages(), package_data={'elastalert': ['schema.yaml']}, install_requires=[ + 'apscheduler>=3.3.0' 'aws-requests-auth>=0.3.0', 'blist>=1.3.6', 'boto3>=1.4.4', diff --git a/tests/base_test.py b/tests/base_test.py index b10eb5a74..fa9018b11 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -88,9 +88,9 @@ def test_init_rule(ea): def test_query(ea): - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) - ea.current_es.search.assert_called_with(body={ + ea.thread_data.current_es.search.assert_called_with(body={ 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) @@ -98,9 +98,9 @@ def test_query(ea): def test_query_with_fields(ea): ea.rules[0]['_source_enabled'] = False - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) - ea.current_es.search.assert_called_with(body={ + ea.thread_data.current_es.search.assert_called_with(body={ 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}], 'fields': ['@timestamp']}, index='idx', ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) @@ -109,11 +109,11 @@ def test_query_with_fields(ea): def test_query_with_unix(ea): ea.rules[0]['timestamp_type'] = 'unix' ea.rules[0]['dt_to_ts'] = dt_to_unix - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) start_unix = dt_to_unix(START) end_unix = dt_to_unix(END) - ea.current_es.search.assert_called_with( + ea.thread_data.current_es.search.assert_called_with( body={'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) @@ -122,18 +122,18 @@ def test_query_with_unix(ea): def test_query_with_unixms(ea): ea.rules[0]['timestamp_type'] = 'unixms' ea.rules[0]['dt_to_ts'] = dt_to_unixms - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) start_unix = dt_to_unixms(START) end_unix = dt_to_unixms(END) - ea.current_es.search.assert_called_with( + ea.thread_data.current_es.search.assert_called_with( body={'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) def test_no_hits(ea): - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 0 @@ -142,7 +142,7 @@ def test_no_terms_hits(ea): ea.rules[0]['use_terms_query'] = True ea.rules[0]['query_key'] = 'QWERTY' ea.rules[0]['doc_type'] = 'uiop' - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_terms_data.call_count == 0 @@ -150,7 +150,7 @@ def test_no_terms_hits(ea): def test_some_hits(ea): hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) hits_dt = generate_hits([START, END]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 1 ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']]) @@ -162,7 +162,7 @@ def test_some_hits_unix(ea): ea.rules[0]['ts_to_dt'] = unix_to_dt hits = generate_hits([dt_to_unix(START), dt_to_unix(END)]) hits_dt = generate_hits([START, END]) - ea.current_es.search.return_value = copy.deepcopy(hits) + ea.thread_data.current_es.search.return_value = copy.deepcopy(hits) ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 1 ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']]) @@ -176,7 +176,7 @@ def _duplicate_hits_generator(timestamps, **kwargs): def test_duplicate_timestamps(ea): - ea.current_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP] * 3, blah='duplicate') + ea.thread_data.current_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP] * 3, blah='duplicate') ea.run_query(ea.rules[0], START, ts_to_dt('2014-01-01T00:00:00Z')) assert len(ea.rules[0]['type'].add_data.call_args_list[0][0][0]) == 3 @@ -189,7 +189,7 @@ def test_duplicate_timestamps(ea): def test_match(ea): hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.run_rule(ea.rules[0], END, START) @@ -280,7 +280,7 @@ def test_match_with_module_with_agg(ea): ea.rules[0]['match_enhancements'] = [mod] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15) hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.run_rule(ea.rules[0], END, START) @@ -294,7 +294,7 @@ def test_match_with_enhancements_first(ea): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15) ea.rules[0]['run_enhancements_first'] = True hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch.object(ea, 'add_aggregated_alert') as add_alert: @@ -317,7 +317,7 @@ def test_agg_matchtime(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10)) hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): # Aggregate first two, query over full range ea.rules[0]['aggregate_by_match_time'] = True @@ -373,7 +373,7 @@ def test_agg_not_matchtime(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] match_time = ts_to_dt('2014-09-26T12:55:00Z') hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch('elastalert.elastalert.ts_now', return_value=match_time): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) @@ -402,7 +402,7 @@ def test_agg_cron(ea): ea.max_aggregation = 1337 hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits alerttime1 = dt_to_ts(ts_to_dt('2014-09-26T12:46:00')) alerttime2 = dt_to_ts(ts_to_dt('2014-09-26T13:04:00')) @@ -439,7 +439,7 @@ def test_agg_no_writeback_connectivity(ea): run again, that they will be passed again to add_aggregated_alert """ hit1, hit2, hit3 = '2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45' hits = generate_hits([hit1, hit2, hit3]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': hit1}, {'@timestamp': hit2}, @@ -453,7 +453,7 @@ def test_agg_no_writeback_connectivity(ea): {'@timestamp': hit2, 'num_hits': 0, 'num_matches': 3}, {'@timestamp': hit3, 'num_hits': 0, 'num_matches': 3}] - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.add_aggregated_alert = mock.Mock() with mock.patch('elastalert.elastalert.elasticsearch_client'): @@ -469,7 +469,7 @@ def test_agg_with_aggregation_key(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:43:45'] match_time = ts_to_dt('2014-09-26T12:45:00Z') hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch('elastalert.elastalert.ts_now', return_value=match_time): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) @@ -562,7 +562,7 @@ def test_compound_query_key(ea): ea.rules[0]['query_key'] = 'this,that,those' ea.rules[0]['compound_query_key'] = ['this', 'that', 'those'] hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that=u'☃', those=4) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) call_args = ea.rules[0]['type'].add_data.call_args_list[0] assert 'this,that,those' in call_args[0][0][0] @@ -604,7 +604,7 @@ def test_silence_query_key(ea): def test_realert(ea): hits = ['2014-09-26T12:35:%sZ' % (x) for x in range(60)] matches = [{'@timestamp': x} for x in hits] - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.rules[0]['realert'] = datetime.timedelta(seconds=50) ea.rules[0]['type'].matches = matches @@ -703,7 +703,7 @@ def test_count(ea): query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end) query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gt'] = dt_to_ts(start) start = start + ea.run_every - ea.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True) + ea.thread_data.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True) def run_and_assert_segmented_queries(ea, start, end, segment_size): @@ -727,8 +727,8 @@ def run_and_assert_segmented_queries(ea, start, end, segment_size): def test_query_segmenting_reset_num_hits(ea): # Tests that num_hits gets reset every time run_query is run def assert_num_hits_reset(): - assert ea.num_hits == 0 - ea.num_hits += 10 + assert ea.thread_data.num_hits == 0 + ea.thread_data.num_hits += 10 with mock.patch.object(ea, 'run_query') as mock_run_query: mock_run_query.side_effect = assert_num_hits_reset() ea.run_rule(ea.rules[0], END, START) @@ -915,6 +915,7 @@ def test_kibana_dashboard(ea): def test_rule_changes(ea): + re = datetime.timedelta(minutes=10) ea.rule_hashes = {'rules/rule1.yaml': 'ABC', 'rules/rule2.yaml': 'DEF'} ea.rules = [ea.init_rule(rule, True) for rule in [{'rule_file': 'rules/rule1.yaml', 'name': 'rule1', 'filter': []}, @@ -926,8 +927,8 @@ def test_rule_changes(ea): with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: with mock.patch('elastalert.elastalert.load_configuration') as mock_load: - mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml'}, - {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml'}] + mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml', 'run_every': re}, + {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml', 'run_every': re}] mock_hashes.return_value = new_hashes ea.load_rule_changes() @@ -1004,9 +1005,9 @@ def test_count_keys(ea): ea.rules[0]['doc_type'] = 'blah' buckets = [{'aggregations': {'filtered': {'counts': {'buckets': [{'key': 'a', 'doc_count': 10}, {'key': 'b', 'doc_count': 5}]}}}}, {'aggregations': {'filtered': {'counts': {'buckets': [{'key': 'd', 'doc_count': 10}, {'key': 'c', 'doc_count': 12}]}}}}] - ea.current_es.search.side_effect = buckets + ea.thread_data.current_es.search.side_effect = buckets counts = ea.get_top_counts(ea.rules[0], START, END, ['this', 'that']) - calls = ea.current_es.search.call_args_list + calls = ea.thread_data.current_es.search.call_args_list assert calls[0][1]['search_type'] == 'count' assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5} assert counts['top_events_this'] == {'a': 10, 'b': 5} @@ -1131,7 +1132,7 @@ def mock_loop(): ea.stop() with mock.patch.object(ea, 'sleep_for', return_value=None): - with mock.patch.object(ea, 'run_all_rules') as mock_run: + with mock.patch.object(ea, 'sleep_for') as mock_run: mock_run.side_effect = mock_loop() start_thread = threading.Thread(target=ea.start) # Set as daemon to prevent a failed test from blocking exit diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..bf066122a 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- import datetime - import logging -import mock import os + +import mock import pytest import elastalert.elastalert @@ -87,7 +87,8 @@ def ea(): 'max_query_size': 10000, 'ts_to_dt': ts_to_dt, 'dt_to_ts': dt_to_ts, - '_source_enabled': True}] + '_source_enabled': True, + 'run_every': datetime.timedelta(seconds=15)}] conf = {'rules_folder': 'rules', 'run_every': datetime.timedelta(minutes=10), 'buffer_time': datetime.timedelta(minutes=5), @@ -103,14 +104,18 @@ def ea(): elastalert.elastalert.elasticsearch_client = mock_es_client with mock.patch('elastalert.elastalert.get_rule_hashes'): with mock.patch('elastalert.elastalert.load_rules') as load_conf: - load_conf.return_value = conf - ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) + with mock.patch('elastalert.elastalert.BackgroundScheduler'): + load_conf.return_value = conf + ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) ea.rules[0]['type'] = mock_ruletype() ea.rules[0]['alert'] = [mock_alert()] ea.writeback_es = mock_es_client() ea.writeback_es.search.return_value = {'hits': {'hits': []}} ea.writeback_es.index.return_value = {'_id': 'ABCD'} ea.current_es = mock_es_client('', '') + ea.thread_data.current_es = ea.current_es + ea.thread_data.num_hits = 0 + ea.thread_data.num_dupes = 0 return ea From c5934546b640d5882529af7ecefea5fb6614807e Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 9 Nov 2018 14:37:19 -0800 Subject: [PATCH 097/264] Version 0.2.0b1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index b2528e3b4..0fe5b5f32 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.37', + version='0.2.0b1', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From 08b42684103a6749897876161067f36044c14cc2 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 9 Nov 2018 16:08:49 -0800 Subject: [PATCH 098/264] Added a message if there are no results when using count-only --- elastalert/test_rule.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 3321d5495..0bbd10a69 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -3,6 +3,7 @@ from __future__ import absolute_import from __future__ import print_function +import argparse import copy import datetime import json @@ -13,7 +14,6 @@ import string import sys -import argparse import mock import yaml @@ -62,7 +62,6 @@ def test_file(self, conf, args): if args.stop_error: exit(1) return None - start_time = ts_now() - datetime.timedelta(days=args.days) end_time = ts_now() ts = conf.get('timestamp_field', '@timestamp') @@ -87,6 +86,7 @@ def test_file(self, conf, args): return None num_hits = len(res['hits']['hits']) if not num_hits: + print("Didn't get any results.") return [] terms = res['hits']['hits'][0]['_source'] From bf385934c424955b22b3e49beeb1b244247238a9 Mon Sep 17 00:00:00 2001 From: Abhishek Jaisingh Date: Thu, 1 Nov 2018 18:50:18 +0530 Subject: [PATCH 099/264] add: custom timeframe testing support --- elastalert/test_rule.py | 13 +++++++++++-- 1 file changed, 11 insertions(+), 2 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 3321d5495..41b55dd95 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -274,7 +274,16 @@ def get_id(): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (args.start)) exit(1) else: - starttime = endtime - datetime.timedelta(days=args.days) + # if days given as command line argument + if args.days > 0: + starttime = endtime - datetime.timedelta(days=args.days) + else: + # if timeframe is given in rule + if 'timeframe' in rule: + starttime = endtime - datetime.timedelta(seconds=rule['timeframe'].total_seconds() * 1.01) + # default is 1 days / 24 hours + else: + starttime = endtime - datetime.timedelta(days=1) # Set run_every to cover the entire time range unless count query, terms query or agg query used # This is to prevent query segmenting which unnecessarily slows down tests @@ -376,7 +385,7 @@ def run_rule_test(self): parser = argparse.ArgumentParser(description='Validate a rule configuration') parser.add_argument('file', metavar='rule', type=str, help='rule configuration filename') parser.add_argument('--schema-only', action='store_true', help='Show only schema errors; do not run query') - parser.add_argument('--days', type=int, default=1, action='store', help='Query the previous N days with this rule') + parser.add_argument('--days', type=int, default=0, action='store', help='Query the previous N days with this rule') parser.add_argument('--start', dest='start', help='YYYY-MM-DDTHH:MM:SS Start querying from this timestamp.') parser.add_argument('--end', dest='end', help='YYYY-MM-DDTHH:MM:SS Query to this timestamp. (Default: present) ' 'Use "NOW" to start from current time. (Default: present)') From 8699a833302b65fd6515ec25871ef748329d7d4a Mon Sep 17 00:00:00 2001 From: Luke Watson Date: Tue, 13 Nov 2018 11:05:02 -0500 Subject: [PATCH 100/264] Add slack_title option --- docs/source/ruletypes.rst | 4 +++- elastalert/alerts.py | 1 + 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index c6f9e59b3..2ed5796b1 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1629,7 +1629,9 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_alert_fields``: You can add additional fields to your slack alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`. -``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. +``slack_title``: Sets a title for the message, this shows up as a blue text at the start of the message + +``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. Requires slack_title to be set. ``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 16d5f1fab..dab324c11 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1112,6 +1112,7 @@ def __init__(self, rule): if isinstance(self.slack_channel_override, basestring): self.slack_channel_override = [self.slack_channel_override] self.slack_title_link = self.rule.get('slack_title_link', '') + self.slack_title = self.rule.get('slack_title', '') self.slack_emoji_override = self.rule.get('slack_emoji_override', ':ghost:') self.slack_icon_url_override = self.rule.get('slack_icon_url_override', '') self.slack_msg_color = self.rule.get('slack_msg_color', 'danger') From cc6fb7f0b0cb6235fcb8dda820928833073158c7 Mon Sep 17 00:00:00 2001 From: Luke Watson Date: Tue, 13 Nov 2018 11:10:58 -0500 Subject: [PATCH 101/264] Add slack title to body --- elastalert/alerts.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index dab324c11..ef396113e 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1177,6 +1177,9 @@ def alert(self, matches): else: payload['icon_emoji'] = self.slack_emoji_override + if self.slack_title != '': + payload['attachments'][0]['title'] = self.slack_title + if self.slack_title_link != '': payload['attachments'][0]['title_link'] = self.slack_title_link From 2277407f278f6a1c3a504e26d50bc4089ac1253a Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Wed, 14 Nov 2018 14:38:35 +0000 Subject: [PATCH 102/264] * Repaired tests --- elastalert/loaders.py | 9 ++++++++- tests/alerts_test.py | 6 ++++-- 2 files changed, 12 insertions(+), 3 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index c4037e997..3b7b1db4e 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -64,15 +64,18 @@ class RulesLoader(object): 'stride': alerts.StrideAlerter, 'ms_teams': alerts.MsTeamsAlerter, 'slack': alerts.SlackAlerter, + 'mattermost': alerts.MattermostAlerter, 'pagerduty': alerts.PagerDutyAlerter, 'exotel': alerts.ExotelAlerter, 'twilio': alerts.TwilioAlerter, 'victorops': alerts.VictorOpsAlerter, 'telegram': alerts.TelegramAlerter, + 'googlechat': alerts.GoogleChatAlerter, 'gitter': alerts.GitterAlerter, 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, - 'post': alerts.HTTPPostAlerter + 'post': alerts.HTTPPostAlerter, + 'hivealerter': alerts.HiveAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list @@ -108,6 +111,10 @@ def load(self, conf, args=None): for rule_file in rule_files: try: rule = self.load_configuration(rule_file, conf, args) + # A rule failed to load, don't try to process it + if not rule: + logging.error('Invalid rule file skipped: %s' % rule_file) + continue # By setting "is_enabled: False" in rule file, a rule is easily disabled if 'is_enabled' in rule and not rule['is_enabled']: continue diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 8653e5aec..8169601da 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1022,7 +1022,8 @@ def test_slack_uses_custom_timeout(): 'alert': [], 'slack_timeout': 20 } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1155,7 +1156,8 @@ def test_slack_uses_list_of_custom_slack_channel(): 'slack_channel_override': ['#test-alert', '#test-alert2'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', From eb4834e55aeab7a9cbf651e190815e3d31448aaa Mon Sep 17 00:00:00 2001 From: Austin Date: Fri, 16 Nov 2018 09:28:44 -0800 Subject: [PATCH 103/264] Adds PagerTree Alerter --- README.md | 3 ++- docs/source/ruletypes.rst | 9 +++++++++ elastalert/alerts.py | 30 ++++++++++++++++++++++++++++++ elastalert/config.py | 1 + elastalert/schema.yaml | 4 ++++ 5 files changed, 46 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 2b09e7d44..e56e7c83b 100644 --- a/README.md +++ b/README.md @@ -51,6 +51,7 @@ Currently, we have built-in support for the following alert types: - AWS SNS - VictorOps - PagerDuty +- PagerTree - Exotel - Twilio - Gitter @@ -93,7 +94,7 @@ Eg: ``--rule this_rule.yaml`` Available at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin). ### Docker -A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`. +A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`. ```bash git clone https://github.com/bitsensor/elastalert.git; cd elastalert diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index c6f9e59b3..32c1ed74e 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1752,6 +1752,15 @@ See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 ``pagerduty_v2_payload_source_args``: If set, and ``pagerduty_v2_payload_source`` is a formattable string, Elastalert will format the source based on the provided array of fields from the rule or match. +PagerTree +~~~~~~~~~ + +PagerTree alerter will trigger an incident to a predefined PagerTree integration url. + +The alerter requires the following options: + +``pagertree_integration_url``: URL generated by PagerTree for the integration. + Exotel ~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 16d5f1fab..a5023f5b7 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1441,6 +1441,36 @@ def get_info(self): return {'type': 'pagerduty', 'pagerduty_client_name': self.pagerduty_client_name} +class PagerTreeAlerter(Alerter): + """ Creates a PagerTree Incident for each alert """ + required_options = frozenset(['pagertree_integration_url']) + + def __init__(self, rule): + super(PagerTreeAlerter, self).__init__(rule) + self.url = self.rule['pagertree_integration_url'] + + def alert(self, matches): + body = self.create_alert_body(matches) + + # post to pagertree + headers = {'content-type': 'application/json'} + # set https proxy, if it was provided + payload = { + "monitoring_tool": "ElastAlert", + "state_message": body + } + + try: + response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies) + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to PagerTree: %s" % e) + elastalert_logger.info("Trigger sent to PagerTree") + + def get_info(self): + return {'type': 'pagertree', + 'pagertree_integration_url': self.pagertree_integration_url} + class ExotelAlerter(Alerter): required_options = frozenset(['exotel_account_sid', 'exotel_auth_token', 'exotel_to_number', 'exotel_from_number']) diff --git a/elastalert/config.py b/elastalert/config.py index d1c27f44a..2a019f4e8 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -75,6 +75,7 @@ 'slack': alerts.SlackAlerter, 'mattermost': alerts.MattermostAlerter, 'pagerduty': alerts.PagerDutyAlerter, + 'pagertree': alerts.PagerTreeAlerter, 'exotel': alerts.ExotelAlerter, 'twilio': alerts.TwilioAlerter, 'victorops': alerts.VictorOpsAlerter, diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 5e3288d6d..0189f07be 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -264,6 +264,10 @@ properties: pagerduty_client_name: {type: string} pagerduty_event_type: {enum: [none, trigger, resolve, acknowledge]} +### PagerTree + pagertree_integration_url: {type: string} + + ### Exotel exotel_account_sid: {type: string} exotel_auth_token: {type: string} From 95ab9df05e16480f3b36f19669111e7f60a180eb Mon Sep 17 00:00:00 2001 From: Austin Date: Fri, 16 Nov 2018 13:31:33 -0800 Subject: [PATCH 104/264] Add the proxy --- elastalert/alerts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index a5023f5b7..5ac104324 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1448,6 +1448,7 @@ class PagerTreeAlerter(Alerter): def __init__(self, rule): super(PagerTreeAlerter, self).__init__(rule) self.url = self.rule['pagertree_integration_url'] + self.pagertree_proxy = self.rule.get('pagertree_proxy', None) def alert(self, matches): body = self.create_alert_body(matches) @@ -1455,6 +1456,7 @@ def alert(self, matches): # post to pagertree headers = {'content-type': 'application/json'} # set https proxy, if it was provided + proxies = {'https': self.pagertree_proxy} if self.pagertree_proxy else None payload = { "monitoring_tool": "ElastAlert", "state_message": body From 32cd8d74f2edbd848b6aa1cbbac674d8e3a6cb78 Mon Sep 17 00:00:00 2001 From: Austin Date: Fri, 16 Nov 2018 13:44:22 -0800 Subject: [PATCH 105/264] Fix the PagerTree url --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 5ac104324..8a23a65fb 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1471,7 +1471,7 @@ def alert(self, matches): def get_info(self): return {'type': 'pagertree', - 'pagertree_integration_url': self.pagertree_integration_url} + 'pagertree_integration_url': self.url} class ExotelAlerter(Alerter): From 530d239d9cffb6285092b2a4417051052a622b84 Mon Sep 17 00:00:00 2001 From: Austin Date: Fri, 16 Nov 2018 14:05:51 -0800 Subject: [PATCH 106/264] Format for PagerTree webhook format --- elastalert/alerts.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 8a23a65fb..bfe21fb72 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1451,15 +1451,15 @@ def __init__(self, rule): self.pagertree_proxy = self.rule.get('pagertree_proxy', None) def alert(self, matches): - body = self.create_alert_body(matches) - # post to pagertree headers = {'content-type': 'application/json'} # set https proxy, if it was provided proxies = {'https': self.pagertree_proxy} if self.pagertree_proxy else None payload = { - "monitoring_tool": "ElastAlert", - "state_message": body + "event_type": "create", + "Id": str(uuid.uuid4())[0:6], + "Title": self.create_title(matches), + "Description": self.create_alert_body(matches) } try: From 9382886f3d714965fc0f212c6701752f4de3eaf9 Mon Sep 17 00:00:00 2001 From: Austin Date: Fri, 16 Nov 2018 14:12:18 -0800 Subject: [PATCH 107/264] use the whole uuid for pagertree id --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index bfe21fb72..d09e71fda 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1457,7 +1457,7 @@ def alert(self, matches): proxies = {'https': self.pagertree_proxy} if self.pagertree_proxy else None payload = { "event_type": "create", - "Id": str(uuid.uuid4())[0:6], + "Id": str(uuid.uuid4()), "Title": self.create_title(matches), "Description": self.create_alert_body(matches) } From 410736bcd44803104383ae9ba07d77452084d976 Mon Sep 17 00:00:00 2001 From: Austin Date: Fri, 16 Nov 2018 14:18:57 -0800 Subject: [PATCH 108/264] Adds changelog for PagerTree addition --- changelog.md | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/changelog.md b/changelog.md index 348e70f74..6ecdf5831 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,10 @@ # Change Log +# v0.1.37 + +### Added +- Added PagerTree alerter + # v0.1.36 ### Added From c162c22ee2f4d57e78398b7bb8ce887d1d09072f Mon Sep 17 00:00:00 2001 From: Austin Date: Fri, 16 Nov 2018 14:32:27 -0800 Subject: [PATCH 109/264] Test want 2 blank lines --- elastalert/alerts.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d09e71fda..2f2757f56 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1441,6 +1441,7 @@ def get_info(self): return {'type': 'pagerduty', 'pagerduty_client_name': self.pagerduty_client_name} + class PagerTreeAlerter(Alerter): """ Creates a PagerTree Incident for each alert """ required_options = frozenset(['pagertree_integration_url']) From adea2dd8d9e51e27b9187cc7a1148ab2ccaa08af Mon Sep 17 00:00:00 2001 From: ryansaunders Date: Wed, 28 Nov 2018 14:02:01 +0000 Subject: [PATCH 110/264] Updated the writeback suffix to include customer id --- elastalert/elastalert.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 78d7525bc..94892d28a 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -200,7 +200,7 @@ def get_index(rule, starttime=None, endtime=None): else: return index - def get_writeback_index(self, doc_type, rule=None): + def get_writeback_index(self, doc_type, rule=None, match_body=None): writeback_index = self.writeback_index if rule is None or 'writeback_suffix' not in rule: if self.is_atleastsix(): @@ -213,8 +213,8 @@ def get_writeback_index(self, doc_type, rule=None): elif doc_type == 'elastalert_error': writeback_index += '_error' else: - suffix = datetime.datetime.utcnow().strftime(rule['writeback_suffix']) - writeback_index += '_' + suffix + suffix = '_' + "{[customers][0][uuid]}_{}".format(match_body, datetime.datetime.utcnow().strftime(rule['writeback_suffix'])) + writeback_index += suffix return writeback_index @@ -1415,7 +1415,7 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No return body def writeback(self, doc_type, body, rule=None): - writeback_index = self.get_writeback_index(doc_type, rule) + writeback_index = self.get_writeback_index(doc_type, rule, body) # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: From e5bb26f7a865fbfc658ba536431e2330dd8e1d5d Mon Sep 17 00:00:00 2001 From: ryansaunders Date: Wed, 28 Nov 2018 14:32:57 +0000 Subject: [PATCH 111/264] writeback suffix is formatted with content from match body --- elastalert/elastalert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 94892d28a..257307b0d 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -213,8 +213,8 @@ def get_writeback_index(self, doc_type, rule=None, match_body=None): elif doc_type == 'elastalert_error': writeback_index += '_error' else: - suffix = '_' + "{[customers][0][uuid]}_{}".format(match_body, datetime.datetime.utcnow().strftime(rule['writeback_suffix'])) - writeback_index += suffix + suffix = datetime.datetime.utcnow().strftime(rule['writeback_suffix'].format(match_body)) + writeback_index += '_' + suffix return writeback_index From 80f27e29e8dbbec5eee9c5c61a099b5a034f772d Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Thu, 29 Nov 2018 08:24:11 +0000 Subject: [PATCH 112/264] Added try-except in case suffix key is not in match_body --- elastalert/elastalert.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 257307b0d..3b6df171f 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -213,8 +213,13 @@ def get_writeback_index(self, doc_type, rule=None, match_body=None): elif doc_type == 'elastalert_error': writeback_index += '_error' else: - suffix = datetime.datetime.utcnow().strftime(rule['writeback_suffix'].format(match_body)) - writeback_index += '_' + suffix + try: + suffix = rule['writeback_suffix'].format(match_body or {}) + suffix = datetime.datetime.utcnow().strftime(suffix) + writeback_index += '_' + suffix + except KeyError as e: + elastalert_logger.critical('Failed to add suffix. Unknown key %s' % str(e)) + pass return writeback_index From 508bf6dde4ae7884965c9f1513f134049fe24037 Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Thu, 29 Nov 2018 15:36:04 +0000 Subject: [PATCH 113/264] Correctly check if template exists and delete --- elastalert/create_index.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index b244693f7..c03922e83 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -228,9 +228,9 @@ def main(): if es_index.exists(index): print('Index ' + index + ' already exists. Skipping index creation.') return None - if es_index.exists_template(index): - print('Template ' + index + ' already exists. Deleting in preparation for creating indexes.') - es_index.delete_template(index) + if es_index.exists_template('elastalert'): + print('Template elastalert already exists. Deleting in preparation for creating indexes.') + es_index.delete_template('elastalert') # (Re-)Create indices. if elasticversion > 5: From 6c540e363344566fca0a86356788ff9e75e9e98c Mon Sep 17 00:00:00 2001 From: Sean Kang Date: Fri, 30 Nov 2018 17:17:29 +1300 Subject: [PATCH 114/264] Add a missing comma to fix dependency error --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 0fe5b5f32..c86885b39 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ packages=find_packages(), package_data={'elastalert': ['schema.yaml']}, install_requires=[ - 'apscheduler>=3.3.0' + 'apscheduler>=3.3.0', 'aws-requests-auth>=0.3.0', 'blist>=1.3.6', 'boto3>=1.4.4', From a6f92e3a6f921af5c9922d8629cf01c4e647c92c Mon Sep 17 00:00:00 2001 From: Peter Scopes Date: Thu, 13 Dec 2018 16:09:02 +0000 Subject: [PATCH 115/264] Completed merge remote-tracking branch 'github/beta' into beta/loader --- elastalert/loaders.py | 2 ++ elastalert/test_rule.py | 1 + elastalert/util.py | 1 + 3 files changed, 4 insertions(+) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 3b7b1db4e..a1cadd342 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -246,6 +246,8 @@ def load_options(self, rule, conf, filename, args=None): rule['query_delay'] = datetime.timedelta(**rule['query_delay']) if 'buffer_time' in rule: rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) + if 'run_every' in rule: + rule['run_every'] = datetime.timedelta(**rule['run_every']) if 'bucket_interval' in rule: rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) if 'exponential_realert' in rule: diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index dba9ae785..350f12d18 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -370,6 +370,7 @@ def run_rule_test(self): 'es_host': 'localhost', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'max_query_size': 10000, 'alert_time_limit': {'hours': 24}, 'old_query_limit': {'weeks': 1}, diff --git a/elastalert/util.py b/elastalert/util.py index ef077c223..dcffc1857 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -20,6 +20,7 @@ def get_module(module_name): """ Loads a module and returns a specific object. module_name should 'module.file.object'. Returns object or raises EAException on error. """ + sys.path.append(os.getcwd()) try: module_path, module_class = module_name.rsplit('.', 1) base_module = __import__(module_path, globals(), locals(), [module_class]) From 4938e4822b4b015e0792a80807080a2e6523b651 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 13 Dec 2018 16:40:11 -0800 Subject: [PATCH 116/264] Fixed a bug causing buffer_time to sometimes be ignored --- elastalert/elastalert.py | 11 +++-------- 1 file changed, 3 insertions(+), 8 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 29cc87568..ed0c06142 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -689,7 +689,6 @@ def get_starttime(self, rule): def set_starttime(self, rule, endtime): """ Given a rule and an endtime, sets the appropriate starttime for it. """ - # This means we are starting fresh if 'starttime' not in rule: if not rule.get('scan_entire_timeframe'): @@ -715,12 +714,9 @@ def set_starttime(self, rule, endtime): if 'minimum_starttime' in rule and rule['minimum_starttime'] > buffer_delta: rule['starttime'] = rule['minimum_starttime'] # If buffer_time doesn't bring us past the previous endtime, use that instead - elif 'previous_endtime' in rule: - if rule['previous_endtime'] < buffer_delta: - rule['starttime'] = rule['previous_endtime'] - self.adjust_start_time_for_overlapping_agg_query(rule) - elif rule.get('allow_buffer_time_overlap'): - rule['starttime'] = buffer_delta + elif 'previous_endtime' in rule and rule['previous_endtime'] < buffer_delta: + rule['starttime'] = rule['previous_endtime'] + self.adjust_start_time_for_overlapping_agg_query(rule) else: rule['starttime'] = buffer_delta @@ -843,7 +839,6 @@ def run_rule(self, rule, endtime, starttime=None): :return: The number of matches that the rule produced. """ run_start = time.time() - self.current_es = elasticsearch_client(rule) self.current_es_addr = (rule['es_host'], rule['es_port']) From 05c65ad6e5998cd61164f7b047b404836051255e Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 13 Dec 2018 17:25:40 -0800 Subject: [PATCH 117/264] Added a unit test to prevent regressions --- tests/base_test.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/tests/base_test.py b/tests/base_test.py index b10eb5a74..a750a633b 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -856,6 +856,12 @@ def test_set_starttime(ea): ea.set_starttime(ea.rules[0], end) assert ea.rules[0]['starttime'] == ea.rules[0]['previous_endtime'] + # Make sure starttime is updated if previous_endtime isn't used + ea.rules[0]['previous_endtime'] = end - ea.buffer_time / 2 + ea.rules[0]['starttime'] = ts_to_dt('2014-10-09T00:00:01') + ea.set_starttime(ea.rules[0], end) + assert ea.rules[0]['starttime'] == end - ea.buffer_time + # scan_entire_timeframe ea.rules[0].pop('previous_endtime') ea.rules[0].pop('starttime') From 67231d349cb361744bc980fbc5d388d3e72e93b4 Mon Sep 17 00:00:00 2001 From: Axel Monroy Date: Fri, 14 Dec 2018 14:04:56 -0700 Subject: [PATCH 118/264] Avoid writing slack_webhook_url in elastalert_status index --- elastalert/alerts.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 16d5f1fab..134914b47 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1198,8 +1198,7 @@ def alert(self, matches): def get_info(self): return {'type': 'slack', - 'slack_username_override': self.slack_username_override, - 'slack_webhook_url': self.slack_webhook_url} + 'slack_username_override': self.slack_username_override} class MattermostAlerter(Alerter): From 061a55a015e9ee9b1125a3d2bd1872a7f6f38e9c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Enrico=20Tr=C3=B6ger?= Date: Thu, 17 Aug 2017 00:28:48 +0200 Subject: [PATCH 119/264] Read logging configuration from config file Support reading a whole dict logging configuration from the config file and configure the logging framework accordingly. Move special logging configuration via command line options (--verbose, --debug) into load_rules() as it needs to be set after the logging framework has been reconfigured. This should keep the previous behavior as close as possible when no logging setup is provided in the configuration and mimic the desired log level adjustments where appropriate even if a custom logging config is given. --- config.yaml.example | 50 ++++++++++++++++++++++++++++++++++++++ docs/source/elastalert.rst | 14 +++++++++++ elastalert/config.py | 37 ++++++++++++++++++++++++++++ elastalert/elastalert.py | 22 ----------------- tests/config_test.py | 1 + 5 files changed, 102 insertions(+), 22 deletions(-) diff --git a/config.yaml.example b/config.yaml.example index beec38030..e39a2bcf8 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -62,3 +62,53 @@ writeback_index: elastalert_status # sending the alert until this time period has elapsed alert_time_limit: days: 2 + +# Custom logging configuration +# If you want to setup your own logging configuration to log into +# files as well or to Logstash and/or modify log levels, use +# the configuration below and adjust to your needs. +# Note: if you run ElastAlert with --verbose/--debug, the log level of +# the "elastalert" logger is changed to INFO, if not already INFO/DEBUG. +#logging: +# version: 1 +# incremental: false +# disable_existing_loggers: false +# formatters: +# logline: +# format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s' +# +# handlers: +# console: +# class: logging.StreamHandler +# formatter: logline +# level: DEBUG +# stream: ext://sys.stderr +# +# file: +# class : logging.FileHandler +# formatter: logline +# level: DEBUG +# filename: elastalert.log +# +# loggers: +# elastalert: +# level: WARN +# handlers: [] +# propagate: true +# +# elasticsearch: +# level: WARN +# handlers: [] +# propagate: true +# +# elasticsearch.trace: +# level: WARN +# handlers: [] +# propagate: true +# +# '': # root logger +# level: WARN +# handlers: +# - console +# - file +# propagate: false diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 249ed65ab..ed0877a1c 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -194,6 +194,20 @@ The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticse ``skip_invalid``: If ``True``, skip invalid files instead of exiting. +======= +Logging +------- + +By default, ElastAlert uses a simple basic logging configuration to print log messages to standard error. +You can change the log level to ``INFO`` messages by using the ``--verbose`` or ``--debug`` command line options. + +If you need a more sophisticated logging configuration, you can provide a full logging configuration +in the config file. This way you can also configure logging to a file, to Logstash and +adjust the logging format. + +For details, see the end of ``config.yaml.example`` where you can find an example logging +configuration. + .. _runningelastalert: diff --git a/elastalert/config.py b/elastalert/config.py index d1c27f44a..29390c7f3 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -3,6 +3,7 @@ import datetime import hashlib import logging +import logging.config import os import sys @@ -19,6 +20,7 @@ from util import dt_to_ts_with_format from util import dt_to_unix from util import dt_to_unixms +from util import elastalert_logger from util import EAException from util import ts_to_dt from util import ts_to_dt_with_format @@ -452,6 +454,9 @@ def load_rules(args): conf = yaml_loader(filename) use_rule = args.rule + # init logging from config and set log levels according to command line options + configure_logging(args, conf) + for env_var, conf_var in env_settings.items(): val = env(env_var, None) if val is not None: @@ -509,6 +514,38 @@ def load_rules(args): return conf +def configure_logging(args, conf): + # configure logging from config file if provided + if 'logging' in conf: + # load new logging config + logging.config.dictConfig(conf['logging']) + + if args.verbose and args.debug: + elastalert_logger.info( + "Note: --debug and --verbose flags are set. --debug takes precedent." + ) + + # re-enable INFO log level on elastalert_logger in verbose/debug mode + # (but don't touch it if it is already set to INFO or below by config) + if args.verbose or args.debug: + if elastalert_logger.level > logging.INFO or elastalert_logger.level == logging.NOTSET: + elastalert_logger.setLevel(logging.INFO) + + if args.debug: + elastalert_logger.info( + """Note: In debug mode, alerts will be logged to console but NOT actually sent. + To send them but remain verbose, use --verbose instead.""" + ) + + if not args.es_debug and 'logging' not in conf: + logging.getLogger('elasticsearch').setLevel(logging.WARNING) + + if args.es_debug_trace: + tracer = logging.getLogger('elasticsearch.trace') + tracer.setLevel(logging.INFO) + tracer.addHandler(logging.FileHandler(args.es_debug_trace)) + + def get_rule_hashes(conf, use_rule=None): rule_files = get_file_paths(conf, use_rule) rule_mod_times = {} diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index ed0c06142..1475b2344 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -105,28 +105,6 @@ def __init__(self, args): self.debug = self.args.debug self.verbose = self.args.verbose - if self.verbose and self.debug: - elastalert_logger.info( - "Note: --debug and --verbose flags are set. --debug takes precedent." - ) - - if self.verbose or self.debug: - elastalert_logger.setLevel(logging.INFO) - - if self.debug: - elastalert_logger.info( - """Note: In debug mode, alerts will be logged to console but NOT actually sent. - To send them but remain verbose, use --verbose instead.""" - ) - - if not self.args.es_debug: - logging.getLogger('elasticsearch').setLevel(logging.WARNING) - - if self.args.es_debug_trace: - tracer = logging.getLogger('elasticsearch.trace') - tracer.setLevel(logging.INFO) - tracer.addHandler(logging.FileHandler(self.args.es_debug_trace)) - self.conf = load_rules(self.args) self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] diff --git a/tests/config_test.py b/tests/config_test.py index f444f0e25..25e1062b2 100644 --- a/tests/config_test.py +++ b/tests/config_test.py @@ -45,6 +45,7 @@ test_args.config = 'test_config' test_args.rule = None test_args.debug = False +test_args.es_debug_trace = None def test_import_rules(): From a0a89ca2e35ecda5fefd138ea1fca5abd2347379 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 17 Dec 2018 14:34:11 -0800 Subject: [PATCH 120/264] Version 0.1.38 --- changelog.md | 22 +++++++++++++++++++--- setup.py | 2 +- 2 files changed, 20 insertions(+), 4 deletions(-) diff --git a/changelog.md b/changelog.md index 6ecdf5831..d007a4dee 100644 --- a/changelog.md +++ b/changelog.md @@ -1,19 +1,35 @@ # Change Log -# v0.1.37 +# v0.1.38 ### Added - Added PagerTree alerter +- Added Line alerter +- Added more customizable logging +- Added new logic in test-rule to detemine the default timeframe + +### Fixed +- Fixed an issue causing buffer_time to sometimes be ignored + +# v0.1.37 + +### Added +- Added more options for Opsgenie alerter +- Added more pagerduty options +- Added ability to add metadata to elastalert logs + +### Fixed +- Fixed some documentation to be more clear +- Stop requiring doc_type for metric aggregations +- No longer puts quotes around regex terms in blacklists or whitelists # v0.1.36 ### Added - Added a prefix "metric_" to the key used for metric aggregations to avoid possible conflicts - Added option to skip Alerta certificate validation -- Added option to gracefully fail when loading rules ### Fixed -- No longer puts quotes around regex terms in blacklists or whitelists - Fixed a typo in the documentation for spike rule # v0.1.35 diff --git a/setup.py b/setup.py index 865d7974f..91a514f18 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.37', + version='0.1.38', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From 69d9b9fc367fa10ca0488a0aa57e2428cfa5488a Mon Sep 17 00:00:00 2001 From: Blake Randall Date: Mon, 17 Dec 2018 20:19:59 -0500 Subject: [PATCH 121/264] Fixing, Issue Yelp#1574 --- elastalert/ruletypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 9ac5c58a6..665f7f615 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1088,7 +1088,7 @@ def check_matches_recursive(self, timestamp, query_key, aggregation_data, compou # add compound key to payload to allow alerts to trigger for every unique occurence compound_value = [match_data[key] for key in self.rules['compound_query_key']] - match_data[self.rules['query_key']] = ",".join(compound_value) + match_data[self.rules['query_key']] = ",".join([unicode(value) for value in compound_value]) self.add_match(match_data) From e888b4d251fcbffa16c6c3a14c8a655ee61d249a Mon Sep 17 00:00:00 2001 From: Abhishek Jaisingh Date: Wed, 19 Dec 2018 11:49:21 +0530 Subject: [PATCH 122/264] Fix Docs: Writing Filters, add ES version info Implementing boolean logic in filters is quite different for Elasticsearch versions, mainly 2.x and 5.x. The examples given in the documentation are of Elasticsearch 2.x ( which is quite outdated ). It should be specified in documentation which version of Elasticsearch do these queries support. Users have already faced issues and confusion regarding this #912, #930 --- docs/source/recipes/writing_filters.rst | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/docs/source/recipes/writing_filters.rst b/docs/source/recipes/writing_filters.rst index 5eff77f84..6a2ca3f07 100644 --- a/docs/source/recipes/writing_filters.rst +++ b/docs/source/recipes/writing_filters.rst @@ -97,7 +97,7 @@ For ranges on fields:: Negation, and, or ***************** -Any of the filters can be embedded in ``not``, ``and``, and ``or``:: +For Elasticsearch 2.X, any of the filters can be embedded in ``not``, ``and``, and ``or``:: filter: - or: @@ -113,6 +113,13 @@ Any of the filters can be embedded in ``not``, ``and``, and ``or``:: term: _type: "something" +For Elasticsearch 5.x, this will not work and to implement boolean logic use query strings:: + + filter: + - query: + query_string: + query: "somefield: somevalue OR foo: bar" + Loading Filters Directly From Kibana 3 -------------------------------------- From 2b847867e759f7d2ee0e287cb5a52b3623e213bd Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 19 Dec 2018 10:37:21 -0800 Subject: [PATCH 123/264] Add match timestamp to pagerduty v2 api --- elastalert/alerts.py | 5 ++++- tests/alerts_test.py | 1 + 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 78caead63..4e98b9a9f 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1374,6 +1374,9 @@ def alert(self, matches): }, }, } + match_timestamp = lookup_es_key(matches[0], self.rule.get('timestamp_field', '@timestamp')) + if match_timestamp: + payload['payload']['timestamp'] = match_timestamp else: payload = { 'service_key': self.pagerduty_service_key, @@ -1910,7 +1913,7 @@ def get_json_payload(self, match): 'tags': [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags], 'correlate': [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate], 'attributes': dict(zip(self.attributes_keys, - [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values])), + [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values])), 'rawData': self.create_alert_body([match]), } diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 7925ed5d4..76b414122 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1364,6 +1364,7 @@ def test_pagerduty_alerter_v2(): 'custom_details': { 'information': 'Test PD Rule\n\n@timestamp: 2017-01-01T00:00:00\nsomefield: foobarbaz\n' }, + 'timestamp': '2017-01-01T00:00:00' }, 'event_action': 'trigger', 'dedup_key': '', From 5ef4ed06618c2474c604949a4d14a5563ae308bd Mon Sep 17 00:00:00 2001 From: dylanjf Date: Fri, 28 Dec 2018 17:27:07 -0500 Subject: [PATCH 124/264] support for spike alerts based on single metric aggregations --- elastalert/config.py | 1 + elastalert/ruletypes.py | 82 +++++++++++++++++++ elastalert/schema.yaml | 16 ++++ elastalert/util.py | 1 + .../example_spike_single_metric_agg.yaml | 45 ++++++++++ 5 files changed, 145 insertions(+) create mode 100644 example_rules/example_spike_single_metric_agg.yaml diff --git a/elastalert/config.py b/elastalert/config.py index b5eaca7b7..98b1bbd70 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -60,6 +60,7 @@ 'cardinality': ruletypes.CardinalityRule, 'metric_aggregation': ruletypes.MetricAggregationRule, 'percentage_match': ruletypes.PercentageMatchRule, + 'spike_aggregation': ruletypes.SpikeMetricAggregationRule } # Used to map names of alerts to their classes diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 665f7f615..12930347a 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1102,6 +1102,88 @@ def crossed_thresholds(self, metric_value): return False +class SpikeMetricAggregationRule(BaseAggregationRule, SpikeRule): + """ A rule that matches when there is a spike in an aggregated event compared to its reference point """ + required_options = frozenset(['metric_agg_key', 'metric_agg_type', 'spike_height', 'spike_type']) + allowed_aggregations = frozenset(['min', 'max', 'avg', 'sum', 'cardinality', 'value_count']) + + def __init__(self, *args): + # We inherit everything from BaseAggregation and Spike, overwrite only what we need in functions below + super(SpikeMetricAggregationRule, self).__init__(*args) + + # MetricAgg alert things + self.metric_key = 'metric_' + self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type'] + if not self.rules['metric_agg_type'] in self.allowed_aggregations: + raise EAException("metric_agg_type must be one of %s" % (str(self.allowed_aggregations))) + + # Disabling bucket intervals (doesn't make sense in context of spike to split up your time period) + if self.rules.get('bucket_interval'): + raise EAException("bucket intervals are not supported for spike aggregation alerts") + + self.rules['aggregation_query_element'] = self.generate_aggregation_query() + + def generate_aggregation_query(self): + """Lifted from MetricAggregationRule, added support for scripted fields""" + if self.rules.get('metric_agg_script'): + return {self.metric_key: {self.rules['metric_agg_type']: self.rules['metric_agg_script']}} + return {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}} + + def add_aggregation_data(self, payload): + """ + BaseAggregationRule.add_aggregation_data unpacks our results and runs checks directly against hardcoded cutoffs. + We instead want to use all of our SpikeRule.handle_event inherited logic (current/reference) from + the aggregation's "value" key to determine spikes from aggregations + """ + for timestamp, payload_data in payload.iteritems(): + if 'bucket_aggs' in payload_data: + self.unwrap_term_buckets(timestamp, payload_data['bucket_aggs']) + else: + # no time / term split, just focus on the agg + event = {self.ts_field: timestamp} + agg_value = payload_data[self.metric_key]['value'] + self.handle_event(event, agg_value, 'all') + return + + def unwrap_term_buckets(self, timestamp, term_buckets, qk=[]): + """ + create separate spike event trackers for each term, + handle compound query keys + """ + for term_data in term_buckets['buckets']: + qk.append(term_data['key']) + + # handle compound query keys (nested aggregations) + if term_data.get('bucket_aggs'): + self.unwrap_term_buckets(timestamp, term_data['bucket_aggs'], qk) + # reset the query key to consider the proper depth for N > 2 + del qk[-1] + continue + + qk_str = ','.join(qk) + agg_value = term_data[self.metric_key]['value'] + event = {self.ts_field: timestamp, + self.rules['query_key']: qk_str} + # pass to SpikeRule's tracker + self.handle_event(event, agg_value, qk_str) + + # handle unpack of lowest level + del qk[-1] + return + + def get_match_str(self, match): + """ + Overwrite SpikeRule's message to relate to the aggregation type & field instead of count + """ + message = 'An abnormal {0} of {1} ({2}) occurred around {3}.\n'.format( + self.rules['metric_agg_type'], self.rules['metric_agg_key'], round(match['spike_count'], 2), + pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time')) + ) + message += 'Preceding that time, there was a {0} of {1} of ({2}) within {3}\n\n'.format( + self.rules['metric_agg_type'], self.rules['metric_agg_key'], + round(match['reference_count'], 2), self.rules['timeframe']) + return message + + class PercentageMatchRule(BaseAggregationRule): required_options = frozenset(['match_bucket_filter']) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index c1197c5e5..06f35fb03 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -93,6 +93,22 @@ oneOf: threshold_ref: {type: integer} threshold_cur: {type: integer} + - title: Spike Aggregation + required: [spike_height, spike_type, timeframe] + properties: + type: {enum: [spike_aggregation]} + spike_height: {type: number} + spike_type: {enum: ["up", "down", "both"]} + metric_agg_type: {enum: ["min", "max", "avg", "sum", "cardinality", "value_count"]} + timeframe: *timeframe + use_count_query: {type: boolean} + doc_type: {type: string} + use_terms_query: {type: boolean} + terms_size: {type: integer} + alert_on_new_data: {type: boolean} + threshold_ref: {type: integer} + threshold_cur: {type: integer} + - title: Flatline required: [threshold, timeframe] properties: diff --git a/elastalert/util.py b/elastalert/util.py index 33f0b4e71..2bef20556 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -284,6 +284,7 @@ def elasticsearch_client(conf): """ returns an Elasticsearch instance configured using an es_conn_config """ es_conn_conf = build_es_conn_config(conf) auth = Auth() + es_conn_conf['http_auth'] = auth(host=es_conn_conf['es_host'], username=es_conn_conf['es_username'], password=es_conn_conf['es_password'], diff --git a/example_rules/example_spike_single_metric_agg.yaml b/example_rules/example_spike_single_metric_agg.yaml new file mode 100644 index 000000000..1d6e247bc --- /dev/null +++ b/example_rules/example_spike_single_metric_agg.yaml @@ -0,0 +1,45 @@ +name: Metricbeat Average CPU Spike Rule +type: metric_aggregation + +#es_host: localhost +#es_port: 9200 + +index: metricbeat-* + +buffer_time: + hours: 1 + +metric_agg_key: system.cpu.user.pct +metric_agg_type: avg +query_key: beat.hostname +doc_type: metricsets + +#allow_buffer_time_overlap: true +#use_run_every_query_size: true + +# (Required one of _cur or _ref, spike specific) +# The minimum number of events that will trigger an alert +# For example, if there are only 2 events between 12:00 and 2:00, and 20 between 2:00 and 4:00 +# _ref is 2 and _cur is 20, and the alert WILL fire because 20 is greater than threshold_cur and (_ref * spike_height) +threshold_cur: 0.9 + +# (Required, spike specific) +# The spike rule matches when the current window contains spike_height times more +# events than the reference window +spike_height: 2 + +# (Required, spike specific) +# The direction of the spike +# 'up' matches only spikes, 'down' matches only troughs +# 'both' matches both spikes and troughs +spike_type: "up" + +filter: +- term: + metricset.name: cpu + +# (Required) +# The alert is use when a match is found +alert: +- "debug" + From 0d2ea75109e86e63b2c643908081302b1a4151b8 Mon Sep 17 00:00:00 2001 From: dylanjf Date: Fri, 28 Dec 2018 17:41:12 -0500 Subject: [PATCH 125/264] tidy up example rule, revert unnecessary return --- elastalert/util.py | 1 - example_rules/example_spike_single_metric_agg.yaml | 10 ++++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/elastalert/util.py b/elastalert/util.py index 2bef20556..33f0b4e71 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -284,7 +284,6 @@ def elasticsearch_client(conf): """ returns an Elasticsearch instance configured using an es_conn_config """ es_conn_conf = build_es_conn_config(conf) auth = Auth() - es_conn_conf['http_auth'] = auth(host=es_conn_conf['es_host'], username=es_conn_conf['es_username'], password=es_conn_conf['es_password'], diff --git a/example_rules/example_spike_single_metric_agg.yaml b/example_rules/example_spike_single_metric_agg.yaml index 1d6e247bc..19753a550 100644 --- a/example_rules/example_spike_single_metric_agg.yaml +++ b/example_rules/example_spike_single_metric_agg.yaml @@ -1,5 +1,5 @@ name: Metricbeat Average CPU Spike Rule -type: metric_aggregation +type: spike_aggregation #es_host: localhost #es_port: 9200 @@ -18,9 +18,11 @@ doc_type: metricsets #use_run_every_query_size: true # (Required one of _cur or _ref, spike specific) -# The minimum number of events that will trigger an alert -# For example, if there are only 2 events between 12:00 and 2:00, and 20 between 2:00 and 4:00 -# _ref is 2 and _cur is 20, and the alert WILL fire because 20 is greater than threshold_cur and (_ref * spike_height) +# The minimum value of the aggregation that will trigger the alert +# For example, if we're tracking the average for a metric whose average is 0.4 between 12:00 and 2:00 +# and 0.95 between 2:00 and 4:00 with spike_height set to 2 and threshhold_cur set to 0.9: +# _ref is 0.4 and _cur is 0.95, and the alert WILL fire +# because 0.95 is greater than threshold_cur (0.9) and (_ref * spike_height (.4 * 2)) threshold_cur: 0.9 # (Required, spike specific) From f923202d050973c2436ade04e3d98b6c227eaeba Mon Sep 17 00:00:00 2001 From: dylanjf Date: Fri, 28 Dec 2018 18:23:15 -0500 Subject: [PATCH 126/264] spike aggregation documentation --- docs/source/ruletypes.rst | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 0643e0761..b0baa7ccd 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1097,6 +1097,51 @@ allign with the time elastalert runs, (This both avoid calculations on partial d See: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html#_offset for a more comprehensive explaination. +Spike Aggregation +~~~~~~~~~~~~~~~~~~ + +``spike_aggregation``: This rule matches when the value of a metric within the calculation window is ``spike_height`` times larger or smaller +than during the previous time period. It uses two sliding windows to compare the current and reference metric values. +We will call these two windows "reference" and "current". + +This rule requires: + +``metric_agg_key``: This is the name of the field over which the metric value will be calculated. The underlying type of this field must be +supported by the specified aggregation type. If using a scripted field via ``metric_agg_script``, this is the name for your scripted field + +``metric_agg_type``: The type of metric aggregation to perform on the ``metric_agg_key`` field. This must be one of 'min', 'max', 'avg', +'sum', 'cardinality', 'value_count'. + +``spike_height``: The ratio of the metric value in the last ``timeframe`` to the previous ``timeframe`` that when hit +will trigger an alert. + +``spike_type``: Either 'up', 'down' or 'both'. 'Up' meaning the rule will only match when the metric value is ``spike_height`` times +higher. 'Down' meaning the reference metric value is ``spike_height`` higher than the current metric value. 'Both' will match either. + +``timeframe``: The rule will average out the rate of events over this time period. For example, ``hours: 1`` means that the 'current' +window will span from present to one hour ago, and the 'reference' window will span from one hour ago to two hours ago. The rule +will not be active until the time elapsed from the first event is at least two timeframes. This is to prevent an alert being triggered +before a baseline rate has been established. This can be overridden using ``alert_on_new_data``. + +Optional: + +``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and +evaluated separately against the 'reference'/'current' metric value and ``spike height``. + +``metric_agg_script``: A 'Painless' formatted script describing how to calculate your metric on-the-fly:: + + metric_agg_key: myScriptedMetric + metric_agg_script: + script: doc['term1'].value * doc['term2'].value + +``threshold_ref``: The minimum value of the metric in the reference window for an alert to trigger. For example, if +``spike_height: 3`` and ``threshold_ref: 10``, then the 'reference' window must have a metric value of 10 and the 'current' window at +least three times that for an alert to be triggered. + +``threshold_cur``: The minimum value of the metric in the current window for an alert to trigger. For example, if +``spike_height: 3`` and ``threshold_cur: 60``, then an alert will occur if the current window has a metric value greater than 60 and +the reference window is less than a third of that value. + Percentage Match ~~~~~~~~~~~~~~~~ From ec9c6e27f03b5abc90f5248b8fbfe571c74d895a Mon Sep 17 00:00:00 2001 From: dylanjf Date: Thu, 3 Jan 2019 12:59:49 -0500 Subject: [PATCH 127/264] allow decimal threshholds for spike aggregations --- elastalert/schema.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 06f35fb03..0c618d429 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -106,8 +106,8 @@ oneOf: use_terms_query: {type: boolean} terms_size: {type: integer} alert_on_new_data: {type: boolean} - threshold_ref: {type: integer} - threshold_cur: {type: integer} + threshold_ref: {type: number} + threshold_cur: {type: number} - title: Flatline required: [threshold, timeframe] From f596c168b4eb6570397d90b331b03d84faff6d22 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 3 Jan 2019 13:19:15 -0800 Subject: [PATCH 128/264] Fixed an issue with using nested field_value --- elastalert/ruletypes.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 9ac5c58a6..d3e1c0dbf 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -421,15 +421,14 @@ def add_data(self, data): if qk is None: qk = 'other' if self.field_value is not None: - if self.field_value in event: - count = lookup_es_key(event, self.field_value) - if count is not None: - try: - count = int(count) - except ValueError: - elastalert_logger.warn('{} is not a number: {}'.format(self.field_value, count)) - else: - self.handle_event(event, count, qk) + count = lookup_es_key(event, self.field_value) + if count is not None: + try: + count = int(count) + except ValueError: + elastalert_logger.warn('{} is not a number: {}'.format(self.field_value, count)) + else: + self.handle_event(event, count, qk) else: self.handle_event(event, 1, qk) From 67d67fea091d9cb8805653cd683ec76d60a10569 Mon Sep 17 00:00:00 2001 From: dylanjf Date: Fri, 4 Jan 2019 16:54:12 -0500 Subject: [PATCH 129/264] added min_doc_count to aggregations to allow for count-based threshholds on query_keys --- docs/source/ruletypes.rst | 12 +++++++++--- elastalert/elastalert.py | 4 +++- elastalert/schema.yaml | 1 + example_rules/example_spike_single_metric_agg.yaml | 9 +++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index b0baa7ccd..113b2e50e 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1078,6 +1078,9 @@ Optional: ``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and evaluated separately against the threshold(s). +``min_doc_count``: The minimum number of events in the current window needed for an alert to trigger. Used in conjunction with ``query_key``, +this will only consider terms which in their last ``buffer_time`` had at least ``min_doc_count`` records. Default 1. + ``use_run_every_query_size``: By default the metric value is calculated over a ``buffer_time`` sized window. If this parameter is true the rule will use ``run_every`` as the calculation window. @@ -1118,7 +1121,7 @@ will trigger an alert. ``spike_type``: Either 'up', 'down' or 'both'. 'Up' meaning the rule will only match when the metric value is ``spike_height`` times higher. 'Down' meaning the reference metric value is ``spike_height`` higher than the current metric value. 'Both' will match either. -``timeframe``: The rule will average out the rate of events over this time period. For example, ``hours: 1`` means that the 'current' +``buffer_time``: The rule will average out the rate of events over this time period. For example, ``hours: 1`` means that the 'current' window will span from present to one hour ago, and the 'reference' window will span from one hour ago to two hours ago. The rule will not be active until the time elapsed from the first event is at least two timeframes. This is to prevent an alert being triggered before a baseline rate has been established. This can be overridden using ``alert_on_new_data``. @@ -1128,11 +1131,11 @@ Optional: ``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and evaluated separately against the 'reference'/'current' metric value and ``spike height``. -``metric_agg_script``: A 'Painless' formatted script describing how to calculate your metric on-the-fly:: +``metric_agg_script``: A `Painless` formatted script describing how to calculate your metric on-the-fly:: metric_agg_key: myScriptedMetric metric_agg_script: - script: doc['term1'].value * doc['term2'].value + script: doc['field1'].value * doc['field2'].value ``threshold_ref``: The minimum value of the metric in the reference window for an alert to trigger. For example, if ``spike_height: 3`` and ``threshold_ref: 10``, then the 'reference' window must have a metric value of 10 and the 'current' window at @@ -1142,6 +1145,9 @@ least three times that for an alert to be triggered. ``spike_height: 3`` and ``threshold_cur: 60``, then an alert will occur if the current window has a metric value greater than 60 and the reference window is less than a third of that value. +``min_doc_count``: The minimum number of events in the current window needed for an alert to trigger. Used in conjunction with ``query_key``, +this will only consider terms which in their last ``buffer_time`` had at least ``min_doc_count`` records. Default 1. + Percentage Match ~~~~~~~~~~~~~~~~ diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 1475b2344..d5bee92ad 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -259,7 +259,9 @@ def get_aggregation_query(self, query, rule, query_key, terms_size, timestamp_fi if query_key is not None: for idx, key in reversed(list(enumerate(query_key.split(',')))): - aggs_element = {'bucket_aggs': {'terms': {'field': key, 'size': terms_size}, 'aggs': aggs_element}} + aggs_element = {'bucket_aggs': {'terms': {'field': key, 'size': terms_size, + 'min_doc_count': rule.get('min_doc_count', 1)}, + 'aggs': aggs_element}} if not rule['five']: query_element['filtered'].update({'aggs': aggs_element}) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 0c618d429..e5caacc6e 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -108,6 +108,7 @@ oneOf: alert_on_new_data: {type: boolean} threshold_ref: {type: number} threshold_cur: {type: number} + min_doc_count: {type: integer} - title: Flatline required: [threshold, timeframe] diff --git a/example_rules/example_spike_single_metric_agg.yaml b/example_rules/example_spike_single_metric_agg.yaml index 19753a550..9248f8fb2 100644 --- a/example_rules/example_spike_single_metric_agg.yaml +++ b/example_rules/example_spike_single_metric_agg.yaml @@ -25,9 +25,14 @@ doc_type: metricsets # because 0.95 is greater than threshold_cur (0.9) and (_ref * spike_height (.4 * 2)) threshold_cur: 0.9 +# (Optional, min_doc_count) +# for rules using a per-term aggregation via query_key, the minimum number of events +# over the past buffer_time needed to update the spike tracker +min_doc_count: 5 + # (Required, spike specific) -# The spike rule matches when the current window contains spike_height times more -# events than the reference window +# The spike aggregation rule matches when the current window contains spike_height times higher aggregated value +# than the reference window spike_height: 2 # (Required, spike specific) From 2e86af0bc2730a48aefe1c6c6c96f291f662dad2 Mon Sep 17 00:00:00 2001 From: dylanjf Date: Fri, 28 Dec 2018 17:27:07 -0500 Subject: [PATCH 130/264] support for spike alerts based on single metric aggregations --- elastalert/config.py | 1 + elastalert/ruletypes.py | 82 +++++++++++++++++++ elastalert/schema.yaml | 16 ++++ elastalert/util.py | 1 + .../example_spike_single_metric_agg.yaml | 45 ++++++++++ 5 files changed, 145 insertions(+) create mode 100644 example_rules/example_spike_single_metric_agg.yaml diff --git a/elastalert/config.py b/elastalert/config.py index b5eaca7b7..98b1bbd70 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -60,6 +60,7 @@ 'cardinality': ruletypes.CardinalityRule, 'metric_aggregation': ruletypes.MetricAggregationRule, 'percentage_match': ruletypes.PercentageMatchRule, + 'spike_aggregation': ruletypes.SpikeMetricAggregationRule } # Used to map names of alerts to their classes diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 7ba113629..e25ef148b 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -1101,6 +1101,88 @@ def crossed_thresholds(self, metric_value): return False +class SpikeMetricAggregationRule(BaseAggregationRule, SpikeRule): + """ A rule that matches when there is a spike in an aggregated event compared to its reference point """ + required_options = frozenset(['metric_agg_key', 'metric_agg_type', 'spike_height', 'spike_type']) + allowed_aggregations = frozenset(['min', 'max', 'avg', 'sum', 'cardinality', 'value_count']) + + def __init__(self, *args): + # We inherit everything from BaseAggregation and Spike, overwrite only what we need in functions below + super(SpikeMetricAggregationRule, self).__init__(*args) + + # MetricAgg alert things + self.metric_key = 'metric_' + self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type'] + if not self.rules['metric_agg_type'] in self.allowed_aggregations: + raise EAException("metric_agg_type must be one of %s" % (str(self.allowed_aggregations))) + + # Disabling bucket intervals (doesn't make sense in context of spike to split up your time period) + if self.rules.get('bucket_interval'): + raise EAException("bucket intervals are not supported for spike aggregation alerts") + + self.rules['aggregation_query_element'] = self.generate_aggregation_query() + + def generate_aggregation_query(self): + """Lifted from MetricAggregationRule, added support for scripted fields""" + if self.rules.get('metric_agg_script'): + return {self.metric_key: {self.rules['metric_agg_type']: self.rules['metric_agg_script']}} + return {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}} + + def add_aggregation_data(self, payload): + """ + BaseAggregationRule.add_aggregation_data unpacks our results and runs checks directly against hardcoded cutoffs. + We instead want to use all of our SpikeRule.handle_event inherited logic (current/reference) from + the aggregation's "value" key to determine spikes from aggregations + """ + for timestamp, payload_data in payload.iteritems(): + if 'bucket_aggs' in payload_data: + self.unwrap_term_buckets(timestamp, payload_data['bucket_aggs']) + else: + # no time / term split, just focus on the agg + event = {self.ts_field: timestamp} + agg_value = payload_data[self.metric_key]['value'] + self.handle_event(event, agg_value, 'all') + return + + def unwrap_term_buckets(self, timestamp, term_buckets, qk=[]): + """ + create separate spike event trackers for each term, + handle compound query keys + """ + for term_data in term_buckets['buckets']: + qk.append(term_data['key']) + + # handle compound query keys (nested aggregations) + if term_data.get('bucket_aggs'): + self.unwrap_term_buckets(timestamp, term_data['bucket_aggs'], qk) + # reset the query key to consider the proper depth for N > 2 + del qk[-1] + continue + + qk_str = ','.join(qk) + agg_value = term_data[self.metric_key]['value'] + event = {self.ts_field: timestamp, + self.rules['query_key']: qk_str} + # pass to SpikeRule's tracker + self.handle_event(event, agg_value, qk_str) + + # handle unpack of lowest level + del qk[-1] + return + + def get_match_str(self, match): + """ + Overwrite SpikeRule's message to relate to the aggregation type & field instead of count + """ + message = 'An abnormal {0} of {1} ({2}) occurred around {3}.\n'.format( + self.rules['metric_agg_type'], self.rules['metric_agg_key'], round(match['spike_count'], 2), + pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time')) + ) + message += 'Preceding that time, there was a {0} of {1} of ({2}) within {3}\n\n'.format( + self.rules['metric_agg_type'], self.rules['metric_agg_key'], + round(match['reference_count'], 2), self.rules['timeframe']) + return message + + class PercentageMatchRule(BaseAggregationRule): required_options = frozenset(['match_bucket_filter']) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index c1197c5e5..06f35fb03 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -93,6 +93,22 @@ oneOf: threshold_ref: {type: integer} threshold_cur: {type: integer} + - title: Spike Aggregation + required: [spike_height, spike_type, timeframe] + properties: + type: {enum: [spike_aggregation]} + spike_height: {type: number} + spike_type: {enum: ["up", "down", "both"]} + metric_agg_type: {enum: ["min", "max", "avg", "sum", "cardinality", "value_count"]} + timeframe: *timeframe + use_count_query: {type: boolean} + doc_type: {type: string} + use_terms_query: {type: boolean} + terms_size: {type: integer} + alert_on_new_data: {type: boolean} + threshold_ref: {type: integer} + threshold_cur: {type: integer} + - title: Flatline required: [threshold, timeframe] properties: diff --git a/elastalert/util.py b/elastalert/util.py index 33f0b4e71..2bef20556 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -284,6 +284,7 @@ def elasticsearch_client(conf): """ returns an Elasticsearch instance configured using an es_conn_config """ es_conn_conf = build_es_conn_config(conf) auth = Auth() + es_conn_conf['http_auth'] = auth(host=es_conn_conf['es_host'], username=es_conn_conf['es_username'], password=es_conn_conf['es_password'], diff --git a/example_rules/example_spike_single_metric_agg.yaml b/example_rules/example_spike_single_metric_agg.yaml new file mode 100644 index 000000000..1d6e247bc --- /dev/null +++ b/example_rules/example_spike_single_metric_agg.yaml @@ -0,0 +1,45 @@ +name: Metricbeat Average CPU Spike Rule +type: metric_aggregation + +#es_host: localhost +#es_port: 9200 + +index: metricbeat-* + +buffer_time: + hours: 1 + +metric_agg_key: system.cpu.user.pct +metric_agg_type: avg +query_key: beat.hostname +doc_type: metricsets + +#allow_buffer_time_overlap: true +#use_run_every_query_size: true + +# (Required one of _cur or _ref, spike specific) +# The minimum number of events that will trigger an alert +# For example, if there are only 2 events between 12:00 and 2:00, and 20 between 2:00 and 4:00 +# _ref is 2 and _cur is 20, and the alert WILL fire because 20 is greater than threshold_cur and (_ref * spike_height) +threshold_cur: 0.9 + +# (Required, spike specific) +# The spike rule matches when the current window contains spike_height times more +# events than the reference window +spike_height: 2 + +# (Required, spike specific) +# The direction of the spike +# 'up' matches only spikes, 'down' matches only troughs +# 'both' matches both spikes and troughs +spike_type: "up" + +filter: +- term: + metricset.name: cpu + +# (Required) +# The alert is use when a match is found +alert: +- "debug" + From fc4a8a5ebf6226902ce22f49c60d485cb038e5ae Mon Sep 17 00:00:00 2001 From: dylanjf Date: Fri, 28 Dec 2018 17:41:12 -0500 Subject: [PATCH 131/264] tidy up example rule, revert unnecessary return --- elastalert/util.py | 1 - example_rules/example_spike_single_metric_agg.yaml | 10 ++++++---- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/elastalert/util.py b/elastalert/util.py index 2bef20556..33f0b4e71 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -284,7 +284,6 @@ def elasticsearch_client(conf): """ returns an Elasticsearch instance configured using an es_conn_config """ es_conn_conf = build_es_conn_config(conf) auth = Auth() - es_conn_conf['http_auth'] = auth(host=es_conn_conf['es_host'], username=es_conn_conf['es_username'], password=es_conn_conf['es_password'], diff --git a/example_rules/example_spike_single_metric_agg.yaml b/example_rules/example_spike_single_metric_agg.yaml index 1d6e247bc..19753a550 100644 --- a/example_rules/example_spike_single_metric_agg.yaml +++ b/example_rules/example_spike_single_metric_agg.yaml @@ -1,5 +1,5 @@ name: Metricbeat Average CPU Spike Rule -type: metric_aggregation +type: spike_aggregation #es_host: localhost #es_port: 9200 @@ -18,9 +18,11 @@ doc_type: metricsets #use_run_every_query_size: true # (Required one of _cur or _ref, spike specific) -# The minimum number of events that will trigger an alert -# For example, if there are only 2 events between 12:00 and 2:00, and 20 between 2:00 and 4:00 -# _ref is 2 and _cur is 20, and the alert WILL fire because 20 is greater than threshold_cur and (_ref * spike_height) +# The minimum value of the aggregation that will trigger the alert +# For example, if we're tracking the average for a metric whose average is 0.4 between 12:00 and 2:00 +# and 0.95 between 2:00 and 4:00 with spike_height set to 2 and threshhold_cur set to 0.9: +# _ref is 0.4 and _cur is 0.95, and the alert WILL fire +# because 0.95 is greater than threshold_cur (0.9) and (_ref * spike_height (.4 * 2)) threshold_cur: 0.9 # (Required, spike specific) From ade12c45ca96b409afe22c3b39bba7f49d2371bb Mon Sep 17 00:00:00 2001 From: dylanjf Date: Fri, 28 Dec 2018 18:23:15 -0500 Subject: [PATCH 132/264] spike aggregation documentation --- docs/source/ruletypes.rst | 45 +++++++++++++++++++++++++++++++++++++++ 1 file changed, 45 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 0643e0761..b0baa7ccd 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1097,6 +1097,51 @@ allign with the time elastalert runs, (This both avoid calculations on partial d See: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html#_offset for a more comprehensive explaination. +Spike Aggregation +~~~~~~~~~~~~~~~~~~ + +``spike_aggregation``: This rule matches when the value of a metric within the calculation window is ``spike_height`` times larger or smaller +than during the previous time period. It uses two sliding windows to compare the current and reference metric values. +We will call these two windows "reference" and "current". + +This rule requires: + +``metric_agg_key``: This is the name of the field over which the metric value will be calculated. The underlying type of this field must be +supported by the specified aggregation type. If using a scripted field via ``metric_agg_script``, this is the name for your scripted field + +``metric_agg_type``: The type of metric aggregation to perform on the ``metric_agg_key`` field. This must be one of 'min', 'max', 'avg', +'sum', 'cardinality', 'value_count'. + +``spike_height``: The ratio of the metric value in the last ``timeframe`` to the previous ``timeframe`` that when hit +will trigger an alert. + +``spike_type``: Either 'up', 'down' or 'both'. 'Up' meaning the rule will only match when the metric value is ``spike_height`` times +higher. 'Down' meaning the reference metric value is ``spike_height`` higher than the current metric value. 'Both' will match either. + +``timeframe``: The rule will average out the rate of events over this time period. For example, ``hours: 1`` means that the 'current' +window will span from present to one hour ago, and the 'reference' window will span from one hour ago to two hours ago. The rule +will not be active until the time elapsed from the first event is at least two timeframes. This is to prevent an alert being triggered +before a baseline rate has been established. This can be overridden using ``alert_on_new_data``. + +Optional: + +``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and +evaluated separately against the 'reference'/'current' metric value and ``spike height``. + +``metric_agg_script``: A 'Painless' formatted script describing how to calculate your metric on-the-fly:: + + metric_agg_key: myScriptedMetric + metric_agg_script: + script: doc['term1'].value * doc['term2'].value + +``threshold_ref``: The minimum value of the metric in the reference window for an alert to trigger. For example, if +``spike_height: 3`` and ``threshold_ref: 10``, then the 'reference' window must have a metric value of 10 and the 'current' window at +least three times that for an alert to be triggered. + +``threshold_cur``: The minimum value of the metric in the current window for an alert to trigger. For example, if +``spike_height: 3`` and ``threshold_cur: 60``, then an alert will occur if the current window has a metric value greater than 60 and +the reference window is less than a third of that value. + Percentage Match ~~~~~~~~~~~~~~~~ From 1a07f51b3ee37f7a61d0edc2844821607e6db2a7 Mon Sep 17 00:00:00 2001 From: dylanjf Date: Thu, 3 Jan 2019 12:59:49 -0500 Subject: [PATCH 133/264] allow decimal threshholds for spike aggregations --- elastalert/schema.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 06f35fb03..0c618d429 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -106,8 +106,8 @@ oneOf: use_terms_query: {type: boolean} terms_size: {type: integer} alert_on_new_data: {type: boolean} - threshold_ref: {type: integer} - threshold_cur: {type: integer} + threshold_ref: {type: number} + threshold_cur: {type: number} - title: Flatline required: [threshold, timeframe] From 382aacd5368b54279be748c2ff42b9cc2a611265 Mon Sep 17 00:00:00 2001 From: dylanjf Date: Fri, 4 Jan 2019 16:54:12 -0500 Subject: [PATCH 134/264] added min_doc_count to aggregations to allow for count-based threshholds on query_keys --- docs/source/ruletypes.rst | 12 +++++++++--- elastalert/elastalert.py | 4 +++- elastalert/schema.yaml | 1 + example_rules/example_spike_single_metric_agg.yaml | 9 +++++++-- 4 files changed, 20 insertions(+), 6 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index b0baa7ccd..113b2e50e 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1078,6 +1078,9 @@ Optional: ``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and evaluated separately against the threshold(s). +``min_doc_count``: The minimum number of events in the current window needed for an alert to trigger. Used in conjunction with ``query_key``, +this will only consider terms which in their last ``buffer_time`` had at least ``min_doc_count`` records. Default 1. + ``use_run_every_query_size``: By default the metric value is calculated over a ``buffer_time`` sized window. If this parameter is true the rule will use ``run_every`` as the calculation window. @@ -1118,7 +1121,7 @@ will trigger an alert. ``spike_type``: Either 'up', 'down' or 'both'. 'Up' meaning the rule will only match when the metric value is ``spike_height`` times higher. 'Down' meaning the reference metric value is ``spike_height`` higher than the current metric value. 'Both' will match either. -``timeframe``: The rule will average out the rate of events over this time period. For example, ``hours: 1`` means that the 'current' +``buffer_time``: The rule will average out the rate of events over this time period. For example, ``hours: 1`` means that the 'current' window will span from present to one hour ago, and the 'reference' window will span from one hour ago to two hours ago. The rule will not be active until the time elapsed from the first event is at least two timeframes. This is to prevent an alert being triggered before a baseline rate has been established. This can be overridden using ``alert_on_new_data``. @@ -1128,11 +1131,11 @@ Optional: ``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and evaluated separately against the 'reference'/'current' metric value and ``spike height``. -``metric_agg_script``: A 'Painless' formatted script describing how to calculate your metric on-the-fly:: +``metric_agg_script``: A `Painless` formatted script describing how to calculate your metric on-the-fly:: metric_agg_key: myScriptedMetric metric_agg_script: - script: doc['term1'].value * doc['term2'].value + script: doc['field1'].value * doc['field2'].value ``threshold_ref``: The minimum value of the metric in the reference window for an alert to trigger. For example, if ``spike_height: 3`` and ``threshold_ref: 10``, then the 'reference' window must have a metric value of 10 and the 'current' window at @@ -1142,6 +1145,9 @@ least three times that for an alert to be triggered. ``spike_height: 3`` and ``threshold_cur: 60``, then an alert will occur if the current window has a metric value greater than 60 and the reference window is less than a third of that value. +``min_doc_count``: The minimum number of events in the current window needed for an alert to trigger. Used in conjunction with ``query_key``, +this will only consider terms which in their last ``buffer_time`` had at least ``min_doc_count`` records. Default 1. + Percentage Match ~~~~~~~~~~~~~~~~ diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 1475b2344..d5bee92ad 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -259,7 +259,9 @@ def get_aggregation_query(self, query, rule, query_key, terms_size, timestamp_fi if query_key is not None: for idx, key in reversed(list(enumerate(query_key.split(',')))): - aggs_element = {'bucket_aggs': {'terms': {'field': key, 'size': terms_size}, 'aggs': aggs_element}} + aggs_element = {'bucket_aggs': {'terms': {'field': key, 'size': terms_size, + 'min_doc_count': rule.get('min_doc_count', 1)}, + 'aggs': aggs_element}} if not rule['five']: query_element['filtered'].update({'aggs': aggs_element}) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 0c618d429..e5caacc6e 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -108,6 +108,7 @@ oneOf: alert_on_new_data: {type: boolean} threshold_ref: {type: number} threshold_cur: {type: number} + min_doc_count: {type: integer} - title: Flatline required: [threshold, timeframe] diff --git a/example_rules/example_spike_single_metric_agg.yaml b/example_rules/example_spike_single_metric_agg.yaml index 19753a550..9248f8fb2 100644 --- a/example_rules/example_spike_single_metric_agg.yaml +++ b/example_rules/example_spike_single_metric_agg.yaml @@ -25,9 +25,14 @@ doc_type: metricsets # because 0.95 is greater than threshold_cur (0.9) and (_ref * spike_height (.4 * 2)) threshold_cur: 0.9 +# (Optional, min_doc_count) +# for rules using a per-term aggregation via query_key, the minimum number of events +# over the past buffer_time needed to update the spike tracker +min_doc_count: 5 + # (Required, spike specific) -# The spike rule matches when the current window contains spike_height times more -# events than the reference window +# The spike aggregation rule matches when the current window contains spike_height times higher aggregated value +# than the reference window spike_height: 2 # (Required, spike specific) From 59b697563f2460ff4067da463bf4611d5f7ed8b2 Mon Sep 17 00:00:00 2001 From: Thomas Morledge Date: Tue, 8 Jan 2019 16:09:40 +0000 Subject: [PATCH 135/264] Changed return codes for test rule --- elastalert/test_rule.py | 28 +++++++++++++++++++++------- 1 file changed, 21 insertions(+), 7 deletions(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 784d7e8a0..8400c89be 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -26,10 +26,19 @@ from elastalert.util import lookup_es_key from elastalert.util import ts_now from elastalert.util import ts_to_dt +from elastalert.util import EAException logging.getLogger().setLevel(logging.INFO) logging.getLogger('elasticsearch').setLevel(logging.WARNING) +""" +Error Codes: + 1: Error connecting to ElasticSearch + 2: Error querying ElasticSearch + 3: Invalid Rule + 4: Missing/invalid timestamp +""" + def print_terms(terms, parent): """ Prints a list of flattened dictionary keys """ @@ -56,6 +65,11 @@ def test_file(self, conf, args): try: ElastAlerter.modify_rule_for_ES5(conf) + except EAException as ea: + print('Invalid filter provided:', str(ea), file=sys.stderr) + if args.stop_error: + exit(3) + return None except Exception as e: print("Error connecting to ElasticSearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) @@ -82,7 +96,7 @@ def test_file(self, conf, args): print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(3) return None num_hits = len(res['hits']['hits']) if not num_hits: @@ -108,7 +122,7 @@ def test_file(self, conf, args): print("Error querying Elasticsearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(2) return None num_hits = res['count'] @@ -152,7 +166,7 @@ def test_file(self, conf, args): print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(2) return None num_hits = len(res['hits']['hits']) @@ -240,7 +254,7 @@ def run_elastalert(self, rule, conf, args): except KeyError as e: print("All documents must have a timestamp and _id: %s" % (e), file=sys.stderr) if args.stop_error: - exit(1) + exit(4) return None # Create mock _id for documents if it's missing @@ -264,7 +278,7 @@ def get_id(): endtime = ts_to_dt(args.end) except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (args.end)) - exit(1) + exit(4) else: endtime = ts_now() if args.start: @@ -272,7 +286,7 @@ def get_id(): starttime = ts_to_dt(args.start) except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (args.start)) - exit(1) + exit(4) else: starttime = endtime - datetime.timedelta(days=args.days) @@ -318,7 +332,7 @@ def get_id(): if call[0][0] == 'elastalert_error': errors = True if errors and args.stop_error: - exit(1) + exit(2) def load_conf(self, rules, args): """ Loads a default conf dictionary (from global config file, if provided, or hard-coded mocked data), From b57d75fc466e9489c744a69c012d4b0332c2f27a Mon Sep 17 00:00:00 2001 From: mnivedithaa Date: Wed, 9 Jan 2019 10:25:59 +0530 Subject: [PATCH 136/264] Using SSL with stomp alerts --- elastalert/alerts.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 78caead63..da44dffd8 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -364,8 +364,9 @@ def alert(self, matches): self.stomp_password = self.rule.get('stomp_password', 'admin') self.stomp_destination = self.rule.get( 'stomp_destination', '/queue/ALERT') + self.stomp_ssl = self.rule.get('stomp_ssl', False) - conn = stomp.Connection([(self.stomp_hostname, self.stomp_hostport)]) + conn = stomp.Connection([(self.stomp_hostname, self.stomp_hostport)],use_ssl=self.stomp_ssl) conn.start() conn.connect(self.stomp_login, self.stomp_password) From ea47c72b2facbe6e1ae680dd1e08a5eee6ed11fb Mon Sep 17 00:00:00 2001 From: mnivedithaa Date: Wed, 9 Jan 2019 11:18:41 +0530 Subject: [PATCH 137/264] Update alerts.py --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index da44dffd8..b31ec039a 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -366,7 +366,7 @@ def alert(self, matches): 'stomp_destination', '/queue/ALERT') self.stomp_ssl = self.rule.get('stomp_ssl', False) - conn = stomp.Connection([(self.stomp_hostname, self.stomp_hostport)],use_ssl=self.stomp_ssl) + conn = stomp.Connection([(self.stomp_hostname, self.stomp_hostport)], use_ssl=self.stomp_ssl) conn.start() conn.connect(self.stomp_login, self.stomp_password) From 707be1db8907775d27e3f3fc5023dfed254f3a24 Mon Sep 17 00:00:00 2001 From: Cyrill Schumacher Date: Thu, 10 Jan 2019 14:37:14 +0100 Subject: [PATCH 138/264] Alerts: create_custom_title can respect a configurable max length For example in Jira a subject is limited to 255 chars. When generating a subject with arguments it sometimes exceeds 255 characters. This feature adds the new configuration parameter "alert_subject_max_len" to limit the length of the custom alert subject to the desired length. Default length is at 2048 chars. --- elastalert/alerts.py | 6 +++++- tests/alerts_test.py | 29 +++++++++++++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index b31ec039a..27e03ac91 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -226,6 +226,7 @@ def create_title(self, matches): def create_custom_title(self, matches): alert_subject = unicode(self.rule['alert_subject']) + alert_subject_max_len = int(self.rule.get('alert_subject_max_len', 2048)) if 'alert_subject_args' in self.rule: alert_subject_args = self.rule['alert_subject_args'] @@ -242,7 +243,10 @@ def create_custom_title(self, matches): missing = self.rule.get('alert_missing_value', '') alert_subject_values = [missing if val is None else val for val in alert_subject_values] - return alert_subject.format(*alert_subject_values) + alert_subject = alert_subject.format(*alert_subject_values) + + if len(alert_subject) > alert_subject_max_len: + alert_subject = alert_subject[:alert_subject_max_len] return alert_subject diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 7925ed5d4..9575a610d 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -2118,3 +2118,32 @@ def test_alerta_new_style(ea): ) assert expected_data == json.loads( mock_post_request.call_args_list[0][1]['data']) + + +def test_alert_subject_size_limit_no_args(ea): + rule = { + 'name': 'test_rule', + 'type': mock_rule(), + 'owner': 'the_owner', + 'priority': 2, + 'alert_subject': 'A very long subject', + 'alert_subject_max_len': 5 + } + alert = Alerter(rule) + alertSubject = alert.create_custom_title([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}]) + assert 5 == len(alertSubject) + + +def test_alert_subject_size_limit_with_args(ea): + rule = { + 'name': 'test_rule', + 'type': mock_rule(), + 'owner': 'the_owner', + 'priority': 2, + 'alert_subject': 'Test alert for {0} {1}', + 'alert_subject_args': ['test_term', 'test.term'], + 'alert_subject_max_len': 6 + } + alert = Alerter(rule) + alertSubject = alert.create_custom_title([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}]) + assert 6 == len(alertSubject) From f7a1cfaa8c5a38ec713312c5f71c579ed9b0fe83 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Nagy=20S=C3=A1ndor?= Date: Tue, 29 Jan 2019 15:36:26 +0100 Subject: [PATCH 139/264] Fixes create_index.py to make it possible to overwrite the config file with --config fixes #2097 create_index config attribute ignored if config.yaml is present --- elastalert/create_index.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index b12ee7e5e..86baf6ab2 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -53,9 +53,7 @@ def main(): parser.add_argument('--recreate', type=bool, default=False, help='Force re-creation of the index (this will cause data loss).') args = parser.parse_args() - if os.path.isfile('config.yaml'): - filename = 'config.yaml' - elif os.path.isfile(args.config): + if os.path.isfile(args.config): filename = args.config else: filename = '' From 9ec5a78443119e326fb1b71a3a6d5a0071091dcb Mon Sep 17 00:00:00 2001 From: Thomas Morledge Date: Mon, 4 Feb 2019 09:06:36 +0000 Subject: [PATCH 140/264] Changed self.smtp.close() to self.smtp.quit() as per documentation --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 27e03ac91..519a8cfc7 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -486,7 +486,7 @@ def alert(self, matches): except SMTPAuthenticationError as e: raise EAException("SMTP username/password rejected: %s" % (e)) self.smtp.sendmail(self.from_addr, to_addr, email_msg.as_string()) - self.smtp.close() + self.smtp.quit() elastalert_logger.info("Sent email to %s" % (to_addr)) From a081c9e53f9626d312eefc557d52b9f659f8c042 Mon Sep 17 00:00:00 2001 From: Thomas Morledge Date: Mon, 4 Feb 2019 09:21:45 +0000 Subject: [PATCH 141/264] Fixed email_tests --- tests/alerts_test.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 9575a610d..603b47eef 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -106,7 +106,7 @@ def test_email(): mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -171,7 +171,7 @@ def test_email_with_unicode_strings(): mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), mock.call().sendmail(mock.ANY, [u'testing@test.test'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -200,7 +200,7 @@ def test_email_with_auth(): mock.call().starttls(certfile=None, keyfile=None), mock.call().login('someone', 'hunter2'), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected @@ -222,7 +222,7 @@ def test_email_with_cert_key(): mock.call().starttls(certfile='dummy/cert.crt', keyfile='dummy/client.key'), mock.call().login('someone', 'hunter2'), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected @@ -240,7 +240,7 @@ def test_email_with_cc(): mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test', 'tester@testing.testing'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -265,7 +265,7 @@ def test_email_with_bcc(): mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test', 'tester@testing.testing'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -300,7 +300,7 @@ def test_email_with_cc_and_bcc(): ], mock.ANY ), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -335,7 +335,7 @@ def test_email_with_args(): mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] From 219ee816f5747df8849bc45277d58e3c73cc482f Mon Sep 17 00:00:00 2001 From: Mehdi Elyassa Date: Tue, 12 Feb 2019 10:42:55 +0100 Subject: [PATCH 142/264] Fix theHive alerter documentation --- docs/source/ruletypes.rst | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 113b2e50e..7dce0b352 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2097,10 +2097,9 @@ Example usage:: hive_host: http://localhost hive_port: hive_apikey: - - hive_proxies: - http: '' - https: '' + hive_proxies: + http: '' + https: '' hive_alert_config: title: 'Title' ## This will default to {rule[index]_rule[name]} if not provided From c16886f3ce752a7edb2f242e6dd18b5139b3127f Mon Sep 17 00:00:00 2001 From: Daniel Browne Date: Fri, 15 Feb 2019 00:36:41 -0600 Subject: [PATCH 143/264] Adding fixes from elastalert_hive_alerter 1.0.0rc2 -Allow observable matches with multiple keys -Allow TheHive custom fields --- elastalert/alerts.py | 18 +++++++++++++++--- 1 file changed, 15 insertions(+), 3 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 27e03ac91..7e311ac44 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2115,7 +2115,8 @@ def alert(self, matches): for mapping in self.rule.get('hive_observable_data_mapping', []): for observable_type, match_data_key in mapping.iteritems(): try: - if match_data_key.replace("{match[", "").replace("]}", "") in context['match']: + match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) + if all([True for k in match_data_keys if k in context['match']]): artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) @@ -2128,19 +2129,30 @@ def alert(self, matches): alert_config.update(self.rule.get('hive_alert_config', {})) for alert_config_field, alert_config_value in alert_config.iteritems(): - if isinstance(alert_config_value, basestring): + if alert_config_field == 'customFields': + custom_fields = CustomFieldHelper() + for cf_key, cf_value in alert_config_value.iteritems(): + try: + func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) + except AttributeError: + raise Exception('unsupported custom field type {}'.format(cf_value['type'])) + value = cf_value['value'].format(**context) + func(cf_key, value) + alert_config[alert_config_field] = custom_fields.build() + elif isinstance(alert_config_value, basestring): alert_config[alert_config_field] = alert_config_value.format(**context) elif isinstance(alert_config_value, (list, tuple)): formatted_list = [] for element in alert_config_value: try: formatted_list.append(element.format(**context)) - except (AttributeError, KeyError): + except (AttributeError, KeyError, IndexError): formatted_list.append(element) alert_config[alert_config_field] = formatted_list alert = Alert(**alert_config) response = api.create_alert(alert) + if response.status_code != 201: raise Exception('alert not successfully created in TheHive\n{}'.format(response.text)) From af9200cd14f8be496d31d771e99752c8c56fa681 Mon Sep 17 00:00:00 2001 From: K Date: Fri, 15 Feb 2019 06:37:11 -0500 Subject: [PATCH 144/264] Correctly set `realert`, `aggregation`, `query_delay` from the config --- elastalert/config.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/elastalert/config.py b/elastalert/config.py index 98b1bbd70..f06c5e70b 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -212,6 +212,10 @@ def load_options(rule, conf, filename, args=None): raise EAException('Invalid time format used: %s' % (e)) # Set defaults, copy defaults from config.yaml + td_fields = ['realert', 'aggregation', 'query_delay'] + for td_field in td_fields: + if td_field in base_config: + rule.setdefault(td_field, datetime.timedelta(**base_config[td_field])) for key, val in base_config.items(): rule.setdefault(key, val) rule.setdefault('name', os.path.splitext(filename)[0]) From ffa7634601ad5e95e2418d4d3921a3751934d99e Mon Sep 17 00:00:00 2001 From: K Date: Fri, 15 Feb 2019 06:45:27 -0500 Subject: [PATCH 145/264] Add exponential_realert to the list of inherited fields --- elastalert/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/config.py b/elastalert/config.py index f06c5e70b..af72e5728 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -212,7 +212,7 @@ def load_options(rule, conf, filename, args=None): raise EAException('Invalid time format used: %s' % (e)) # Set defaults, copy defaults from config.yaml - td_fields = ['realert', 'aggregation', 'query_delay'] + td_fields = ['realert', 'exponential_realert', 'aggregation', 'query_delay'] for td_field in td_fields: if td_field in base_config: rule.setdefault(td_field, datetime.timedelta(**base_config[td_field])) From fc1995c681ebcdef7f83adda9bb23cde26bf2329 Mon Sep 17 00:00:00 2001 From: Daniel Browne Date: Mon, 18 Feb 2019 01:00:34 -0600 Subject: [PATCH 146/264] Adding missing imports --- elastalert/alerts.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 7e311ac44..7e0823276 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -4,6 +4,7 @@ import json import logging import os +import re import subprocess import sys import time @@ -31,6 +32,7 @@ from thehive4py.api import TheHiveApi from thehive4py.models import Alert from thehive4py.models import AlertArtifact +from thehive4py.models import CustomFieldHelper from twilio.base.exceptions import TwilioRestException from twilio.rest import Client as TwilioClient from util import EAException From ff4726febaa8520a94d5dc3631278a431022c4af Mon Sep 17 00:00:00 2001 From: Daniel Browne Date: Wed, 20 Feb 2019 22:10:34 -0600 Subject: [PATCH 147/264] Check for rule matches as well to avoid a potential KeyError --- elastalert/alerts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 7e0823276..0e7c1dd83 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2118,7 +2118,9 @@ def alert(self, matches): for observable_type, match_data_key in mapping.iteritems(): try: match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) - if all([True for k in match_data_keys if k in context['match']]): + rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) + if all([True for k in match_data_keys+rule_data_keys + if k in context['match'].keys()+context['rule'].keys()]): artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) From a85427039031416a5451d37da9f51ae07686d3ac Mon Sep 17 00:00:00 2001 From: Daniel Browne Date: Thu, 21 Feb 2019 17:06:34 -0600 Subject: [PATCH 148/264] Neatening code to pass linting --- elastalert/alerts.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 0e7c1dd83..1fa08ee2f 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2119,8 +2119,9 @@ def alert(self, matches): try: match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) - if all([True for k in match_data_keys+rule_data_keys - if k in context['match'].keys()+context['rule'].keys()]): + data_keys = match_data_keys + rule_data_keys + context_keys = context['match'].keys() + context['rule'].keys() + if all([True for k in data_keys if k in context_keys]): artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) From a8bfdfc9ecc9f03ab3d06567146fcdc2c8c6230a Mon Sep 17 00:00:00 2001 From: K Date: Fri, 22 Feb 2019 14:22:24 -0500 Subject: [PATCH 149/264] Add support for `min_doc_count` to terms queries --- elastalert/elastalert.py | 12 ++++++++---- tests/base_test.py | 2 +- 2 files changed, 9 insertions(+), 5 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index d5bee92ad..22b9d49c8 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -222,17 +222,21 @@ def get_query(filters, starttime=None, endtime=None, sort=True, timestamp_field= query['sort'] = [{timestamp_field: {'order': 'desc' if desc else 'asc'}}] return query - def get_terms_query(self, query, size, field, five=False): + def get_terms_query(self, query, rule, size, field, five=False): """ Takes a query generated by get_query and outputs a aggregation query """ query_element = query['query'] if 'sort' in query_element: query_element.pop('sort') if not five: - query_element['filtered'].update({'aggs': {'counts': {'terms': {'field': field, 'size': size}}}}) + query_element['filtered'].update({'aggs': {'counts': {'terms': {'field': field, + 'size': size, + 'min_doc_count': rule.get('min_doc_count', 1)}}}}) aggs_query = {'aggs': query_element} else: aggs_query = query - aggs_query['aggs'] = {'counts': {'terms': {'field': field, 'size': size}}} + aggs_query['aggs'] = {'counts': {'terms': {'field': field, + 'size': size, + 'min_doc_count': rule.get('min_doc_count', 1)}}} return aggs_query def get_aggregation_query(self, query, rule, query_key, terms_size, timestamp_field='@timestamp'): @@ -485,7 +489,7 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non ) if size is None: size = rule.get('terms_size', 50) - query = self.get_terms_query(base_query, size, key, rule['five']) + query = self.get_terms_query(base_query, rule, size, key, rule['five']) try: if not rule['five']: diff --git a/tests/base_test.py b/tests/base_test.py index a750a633b..4aa2b0cfc 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1014,7 +1014,7 @@ def test_count_keys(ea): counts = ea.get_top_counts(ea.rules[0], START, END, ['this', 'that']) calls = ea.current_es.search.call_args_list assert calls[0][1]['search_type'] == 'count' - assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5} + assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5, 'min_doc_count': 1} assert counts['top_events_this'] == {'a': 10, 'b': 5} assert counts['top_events_that'] == {'d': 10, 'c': 12} From 5d7ee22e0ca09a151126be5e472792a2527d5466 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Fri, 1 Mar 2019 12:01:18 +0100 Subject: [PATCH 150/264] First version of zabbix alert --- README.md | 1 + elastalert/zabbix.py | 102 +++++++++++++++++++++++++++++++++++++++++++ requirements.txt | 1 + 3 files changed, 104 insertions(+) create mode 100644 elastalert/zabbix.py diff --git a/README.md b/README.md index 1bfa1f2ea..ab9451bc4 100644 --- a/README.md +++ b/README.md @@ -56,6 +56,7 @@ Currently, we have built-in support for the following alert types: - Twilio - Gitter - Line Notify +- Zabbix Additional rule types and alerts can be easily imported or written. diff --git a/elastalert/zabbix.py b/elastalert/zabbix.py new file mode 100644 index 000000000..b8dd78408 --- /dev/null +++ b/elastalert/zabbix.py @@ -0,0 +1,102 @@ +from alerts import Alerter, BasicMatchString +import logging +from pprint import pprint +from pyzabbix.api import ZabbixAPI +from pyzabbix import ZabbixSender, ZabbixMetric +from pyzabbix.api import ZabbixAPIException +from requests.packages.urllib3.exceptions import InsecureRequestWarning +from datetime import datetime + + +class ZabbixClient(ZabbixAPI): + + def __init__(self, url='http://localhost', use_authenticate=False, user='Admin', password='zabbix', sender_host='localhost', sender_port=10051): + self.url = url + self.use_authenticate = use_authenticate + self.sender_host = sender_host + self.sender_port = sender_port + self.metrics_chunk_size = 200 + self.aggregated_metrics = [] + self.logger = logging.getLogger(self.__class__.__name__) + super(ZabbixClient, self).__init__(url=self.url, use_authenticate=self.use_authenticate, user=user, password=password) + + def send_metric(self, hostname, key, data): + zm = ZabbixMetric(hostname, key, data) + if self.send_aggregated_metrics: + + self.aggregated_metrics.append(zm) + if len(self.aggregated_metrics) > self.metrics_chunk_size: + self.logger.info("Sending: %s metrics" % (len(self.aggregated_metrics))) + try: + ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port).send(self.aggregated_metrics) + self.aggregated_metrics = [] + except Exception as e: + self.logger.exception(e) + pass + else: + try: + ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port).send(zm) + except Exception as e: + self.logger.exception(e) + pass + +class ZabbixAlerter(Alerter): + + # By setting required_options to a set of strings + # You can ensure that the rule config file specifies all + # of the options. Otherwise, ElastAlert will throw an exception + # when trying to load the rule. + required_options = frozenset(['zbx_sender_host', 'zbx_sender_port', 'zbx_host', 'zbx_key']) + + def __init__(self, *args): + super(ZabbixAlerter, self).__init__(*args) + + self.zbx_sender_host = self.rule.get('zbx_sender_host', 'localhost') + self.zbx_sender_port = self.rule.get('zbx_sender_port', 10051) + self.zbx_host = self.rule.get('zbx_host') + self.zbx_key = self.rule.get('zbx_key') + + + # Alert is called + def alert(self, matches): + + # Matches is a list of match dictionaries. + # It contains more than one match when the alert has + # the aggregation option set + + zm = [] + +# zm = [ZabbixMetric(self.zbx_host, self.zbx_key, matches_len)] +# ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) + + for match in matches: + + ts_epoch = int(datetime.strptime(match['@timestamp'],"%Y-%m-%dT%H:%M:%S.%fZ").strftime('%s')) + zm.append(ZabbixMetric(host=self.zbx_host, key=self.zbx_key, value=1, clock=ts_epoch)) +# +# #if match['operation']: +# # key = '{0}[{1}]'.format(match['operation'], self.zbx_key) + #else: + # continue + #print(key) + +# zm = ZabbixMetric(self.zbx_host, key, 1) +# ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) + + + # Config options can be accessed with self.rule +# with open(self.rule['output_file_path'], "a") as output_file: + + # basic_match_string will transform the match into the default + # human readable string format +# match_string = str(BasicMatchString(self.rule, match)) + +# output_file.write(match_string) + + ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) + + # get_info is called after an alert is sent to get data that is written back + # to Elasticsearch in the field "alert_info" + # It should return a dict of information relevant to what the alert does + def get_info(self): + return {'type': 'zabbix Alerter'} \ No newline at end of file diff --git a/requirements.txt b/requirements.txt index 4f23f2ec1..53c9e5c08 100644 --- a/requirements.txt +++ b/requirements.txt @@ -19,3 +19,4 @@ twilio==6.0.0 thehive4py>=1.4.4 python-magic>=0.4.15 cffi>=1.11.5 +py-zabbix==1.1.3 \ No newline at end of file From 1fe7047e83caae77367206ee58f90ce979ea5fc5 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Fri, 1 Mar 2019 12:16:11 +0100 Subject: [PATCH 151/264] flake8 --- elastalert/zabbix.py | 34 +++++++++++++++++----------------- 1 file changed, 17 insertions(+), 17 deletions(-) diff --git a/elastalert/zabbix.py b/elastalert/zabbix.py index b8dd78408..6af59dedb 100644 --- a/elastalert/zabbix.py +++ b/elastalert/zabbix.py @@ -1,16 +1,17 @@ -from alerts import Alerter, BasicMatchString +from alerts import Alerter#, BasicMatchString import logging -from pprint import pprint +#from pprint import pprint from pyzabbix.api import ZabbixAPI from pyzabbix import ZabbixSender, ZabbixMetric -from pyzabbix.api import ZabbixAPIException -from requests.packages.urllib3.exceptions import InsecureRequestWarning +#from pyzabbix.api import ZabbixAPIException +#from requests.packages.urllib3.exceptions import InsecureRequestWarning from datetime import datetime class ZabbixClient(ZabbixAPI): - def __init__(self, url='http://localhost', use_authenticate=False, user='Admin', password='zabbix', sender_host='localhost', sender_port=10051): + def __init__(self, url='http://localhost', use_authenticate=False, user='Admin', password='zabbix', sender_host='localhost', + sender_port=10051): self.url = url self.use_authenticate = use_authenticate self.sender_host = sender_host @@ -40,6 +41,7 @@ def send_metric(self, hostname, key, data): self.logger.exception(e) pass + class ZabbixAlerter(Alerter): # By setting required_options to a set of strings @@ -50,13 +52,12 @@ class ZabbixAlerter(Alerter): def __init__(self, *args): super(ZabbixAlerter, self).__init__(*args) - + self.zbx_sender_host = self.rule.get('zbx_sender_host', 'localhost') self.zbx_sender_port = self.rule.get('zbx_sender_port', 10051) self.zbx_host = self.rule.get('zbx_host') self.zbx_key = self.rule.get('zbx_key') - # Alert is called def alert(self, matches): @@ -71,24 +72,23 @@ def alert(self, matches): for match in matches: - ts_epoch = int(datetime.strptime(match['@timestamp'],"%Y-%m-%dT%H:%M:%S.%fZ").strftime('%s')) + ts_epoch = int(datetime.strptime(match['@timestamp'], "%Y-%m-%dT%H:%M:%S.%fZ").strftime('%s')) zm.append(ZabbixMetric(host=self.zbx_host, key=self.zbx_key, value=1, clock=ts_epoch)) # -# #if match['operation']: -# # key = '{0}[{1}]'.format(match['operation'], self.zbx_key) - #else: - # continue - #print(key) +# if match['operation']: +# key = '{0}[{1}]'.format(match['operation'], self.zbx_key) +# else: +# continue +# print(key) # zm = ZabbixMetric(self.zbx_host, key, 1) # ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) - # Config options can be accessed with self.rule # with open(self.rule['output_file_path'], "a") as output_file: - # basic_match_string will transform the match into the default - # human readable string format +# # basic_match_string will transform the match into the default +# # human readable string format # match_string = str(BasicMatchString(self.rule, match)) # output_file.write(match_string) @@ -99,4 +99,4 @@ def alert(self, matches): # to Elasticsearch in the field "alert_info" # It should return a dict of information relevant to what the alert does def get_info(self): - return {'type': 'zabbix Alerter'} \ No newline at end of file + return {'type': 'zabbix Alerter'} From 196a23e8f54230be6b4b043f2ceaa400eb109b26 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Fri, 1 Mar 2019 12:27:00 +0100 Subject: [PATCH 152/264] flake8 2 --- elastalert/zabbix.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/elastalert/zabbix.py b/elastalert/zabbix.py index 6af59dedb..5c1a2f2d7 100644 --- a/elastalert/zabbix.py +++ b/elastalert/zabbix.py @@ -1,10 +1,10 @@ -from alerts import Alerter#, BasicMatchString +from alerts import Alerter # , BasicMatchString import logging -#from pprint import pprint +# from pprint import pprint from pyzabbix.api import ZabbixAPI from pyzabbix import ZabbixSender, ZabbixMetric -#from pyzabbix.api import ZabbixAPIException -#from requests.packages.urllib3.exceptions import InsecureRequestWarning +# from pyzabbix.api import ZabbixAPIException +# from requests.packages.urllib3.exceptions import InsecureRequestWarning from datetime import datetime From 7016ce1aa0ea4cfa9cbd624f9eb4df3de39d2fd7 Mon Sep 17 00:00:00 2001 From: Daniel Browne Date: Fri, 8 Mar 2019 10:59:14 -0600 Subject: [PATCH 153/264] Fixing bug where artifact creation would be attempted when requested keys were not present in the match data --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 1fa08ee2f..0e9120311 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2121,7 +2121,7 @@ def alert(self, matches): rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) data_keys = match_data_keys + rule_data_keys context_keys = context['match'].keys() + context['rule'].keys() - if all([True for k in data_keys if k in context_keys]): + if all([True if k in context_keys else False for k in data_keys]): artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) From ed03772b9a4003e38094794bf99d9e4068e276c1 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Tue, 26 Mar 2019 00:20:42 +0100 Subject: [PATCH 154/264] Remove all the commented out imports --- elastalert/zabbix.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/elastalert/zabbix.py b/elastalert/zabbix.py index 5c1a2f2d7..d830ee3b9 100644 --- a/elastalert/zabbix.py +++ b/elastalert/zabbix.py @@ -1,10 +1,7 @@ from alerts import Alerter # , BasicMatchString import logging -# from pprint import pprint from pyzabbix.api import ZabbixAPI from pyzabbix import ZabbixSender, ZabbixMetric -# from pyzabbix.api import ZabbixAPIException -# from requests.packages.urllib3.exceptions import InsecureRequestWarning from datetime import datetime From a8487adb3115ad78f78c0faa7cfe4e40b6e4ac6d Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 26 Mar 2019 16:50:57 -0700 Subject: [PATCH 155/264] Add ability to lookup into arrays --- elastalert/util.py | 52 ++++++++++++++++++++++++++++++++-------------- tests/util_test.py | 18 ++++++++++++++++ 2 files changed, 54 insertions(+), 16 deletions(-) diff --git a/elastalert/util.py b/elastalert/util.py index 33f0b4e71..ee137e2f4 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -3,6 +3,7 @@ import datetime import logging import os +import re import dateutil.parser import dateutil.tz @@ -60,27 +61,46 @@ def _find_es_dict_by_key(lookup_dict, term): # For example: # {'foo.bar': {'bar': 'ray'}} to look up foo.bar will return {'bar': 'ray'}, not 'ray' dict_cursor = lookup_dict - subkeys = term.split('.') - subkey = '' - while len(subkeys) > 0: - if not dict_cursor: - return {}, None + while term: + split_results = re.split(r'\[(\d?)\]', term, 1) + if len(split_results) == 3: + sub_term, index, term = split_results + index = index or '0' + index = int(index) + else: + sub_term, index, term = split_results + [None, ''] - subkey += subkeys.pop(0) + subkeys = sub_term.split('.') - if subkey in dict_cursor: - if len(subkeys) == 0: - break + subkey = '' + + while len(subkeys) > 0: + if not dict_cursor: + return {}, None + subkey += subkeys.pop(0) + + if subkey in dict_cursor: + if len(subkeys) == 0: + break + dict_cursor = dict_cursor[subkey] + subkey = '' + elif len(subkeys) == 0: + # If there are no keys left to match, return None values + dict_cursor = None + subkey = None + else: + subkey += '.' + + if index is not None and subkey: dict_cursor = dict_cursor[subkey] - subkey = '' - elif len(subkeys) == 0: - # If there are no keys left to match, return None values - dict_cursor = None - subkey = None - else: - subkey += '.' + if type(dict_cursor) == list and len(dict_cursor) > index: + subkey = index + if term: + dict_cursor = dict_cursor[subkey] + else: + return {}, None return dict_cursor, subkey diff --git a/tests/util_test.py b/tests/util_test.py index 0f8d1d6a1..5eb7bad71 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -105,6 +105,24 @@ def test_looking_up_nested_composite_keys(ea): assert lookup_es_key(record, 'Fields.ts.value') == expected +def test_looking_up_arrays(ea): + record = { + 'flags': [1, 2, 3], + 'objects': [ + {'foo': 'bar'}, + {'foo': [{'bar': 'baz'}]}, + {'foo': {'bar': 'baz'}} + ] + } + assert lookup_es_key(record, 'flags[]') == 1 + assert lookup_es_key(record, 'flags[1]') == 2 + assert lookup_es_key(record, 'objects[]foo') == 'bar' + assert lookup_es_key(record, 'objects[1]foo[0]bar') == 'baz' + assert lookup_es_key(record, 'objects[2]foo.bar') == 'baz' + assert lookup_es_key(record, 'objects[1]foo[1]bar') is None + assert lookup_es_key(record, 'objects[1]foo[0]baz') is None + + def test_add_raw_postfix(ea): expected = 'foo.raw' assert add_raw_postfix('foo', False) == expected From 1e5d8664d2deddf953784b179d2dd968a78dd258 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 26 Mar 2019 17:40:06 -0700 Subject: [PATCH 156/264] Added documentation --- docs/source/ruletypes.rst | 3 ++- elastalert/util.py | 3 +-- tests/util_test.py | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 113b2e50e..a967f353f 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1312,6 +1312,8 @@ field_values will contain every key value pair included in the results from Elas every key in ``include``, every key in ``top_count_keys``, ``query_key``, and ``compare_key``. If the alert spans multiple events, these values may come from an individual event, usually the one which triggers the alert. +When using ``alert_text_args``, you can access nested fields and index into arrays. For example, if your match was ``{"data": {"ips": ["127.0.0.1", "12.34.56.78"]}}``, then by using ``"data.ips[1]"`` in ``alert_text_args``, it would replace value with ``"12.34.56.78"``. This can go arbitrarily deep into fields and will still work on keys that contain dots themselves. + Command ~~~~~~~ @@ -2117,4 +2119,3 @@ Example usage:: - domain: "{match[field1]}_{rule[name]}" - domain: "{match[field]}" - ip: "{match[ip_field]}" - diff --git a/elastalert/util.py b/elastalert/util.py index ee137e2f4..d24256994 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -63,10 +63,9 @@ def _find_es_dict_by_key(lookup_dict, term): dict_cursor = lookup_dict while term: - split_results = re.split(r'\[(\d?)\]', term, 1) + split_results = re.split(r'\[(\d)\]', term, maxsplit=1) if len(split_results) == 3: sub_term, index, term = split_results - index = index or '0' index = int(index) else: sub_term, index, term = split_results + [None, ''] diff --git a/tests/util_test.py b/tests/util_test.py index 5eb7bad71..3e2a8cbb3 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -114,9 +114,9 @@ def test_looking_up_arrays(ea): {'foo': {'bar': 'baz'}} ] } - assert lookup_es_key(record, 'flags[]') == 1 + assert lookup_es_key(record, 'flags[0]') == 1 assert lookup_es_key(record, 'flags[1]') == 2 - assert lookup_es_key(record, 'objects[]foo') == 'bar' + assert lookup_es_key(record, 'objects[0]foo') == 'bar' assert lookup_es_key(record, 'objects[1]foo[0]bar') == 'baz' assert lookup_es_key(record, 'objects[2]foo.bar') == 'baz' assert lookup_es_key(record, 'objects[1]foo[1]bar') is None From 9c27c885106c2cf46097e17ab636c7a35de3f4d2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Wed, 27 Mar 2019 09:00:59 +0100 Subject: [PATCH 157/264] Point travis repo status badge to matsgoran/elastalert --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 1bfa1f2ea..0900957e9 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![Stories in Ready](https://badge.waffle.io/Yelp/elastalert.png?label=ready&title=Ready)](https://waffle.io/Yelp/elastalert) [![Stories in In Progress](https://badge.waffle.io/Yelp/elastalert.png?label=in%20progress&title=In%20Progress)](https://waffle.io/Yelp/elastalert) -[![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) +[![Build Status](https://travis-ci.com/matsgoran/elastalert.svg?branch=master)](https://travis-ci.com/matsgoran/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) ## ElastAlert - [Read the Docs](http://elastalert.readthedocs.org). From bc4e2040606abe79aea42dd68f50e51aea00a890 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Wed, 27 Mar 2019 10:17:47 +0100 Subject: [PATCH 158/264] Move json mappings into files --- elastalert/es_mappings/5/elastalert.json | 28 +++++++++++++++++ .../es_mappings/5/elastalert_error.json | 14 +++++++++ .../es_mappings/5/elastalert_status.json | 13 ++++++++ elastalert/es_mappings/5/past_elastalert.json | 20 +++++++++++++ elastalert/es_mappings/5/silence.json | 17 +++++++++++ elastalert/es_mappings/6/elastalert.json | 30 +++++++++++++++++++ .../es_mappings/6/elastalert_error.json | 14 +++++++++ .../es_mappings/6/elastalert_status.json | 14 +++++++++ elastalert/es_mappings/6/past_elastalert.json | 22 ++++++++++++++ elastalert/es_mappings/6/silence.json | 18 +++++++++++ 10 files changed, 190 insertions(+) create mode 100644 elastalert/es_mappings/5/elastalert.json create mode 100644 elastalert/es_mappings/5/elastalert_error.json create mode 100644 elastalert/es_mappings/5/elastalert_status.json create mode 100644 elastalert/es_mappings/5/past_elastalert.json create mode 100644 elastalert/es_mappings/5/silence.json create mode 100644 elastalert/es_mappings/6/elastalert.json create mode 100644 elastalert/es_mappings/6/elastalert_error.json create mode 100644 elastalert/es_mappings/6/elastalert_status.json create mode 100644 elastalert/es_mappings/6/past_elastalert.json create mode 100644 elastalert/es_mappings/6/silence.json diff --git a/elastalert/es_mappings/5/elastalert.json b/elastalert/es_mappings/5/elastalert.json new file mode 100644 index 000000000..989769459 --- /dev/null +++ b/elastalert/es_mappings/5/elastalert.json @@ -0,0 +1,28 @@ +{ + "elastalert": { + "properties": { + "rule_name": { + "type": "keyword" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "alert_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_body": { + "type": "object", + "enabled": "false" + }, + "aggregate_id": { + "type": "keyword" + } + } + } +} \ No newline at end of file diff --git a/elastalert/es_mappings/5/elastalert_error.json b/elastalert/es_mappings/5/elastalert_error.json new file mode 100644 index 000000000..8672dbd94 --- /dev/null +++ b/elastalert/es_mappings/5/elastalert_error.json @@ -0,0 +1,14 @@ +{ + "elastalert_error": { + "properties": { + "data": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} \ No newline at end of file diff --git a/elastalert/es_mappings/5/elastalert_status.json b/elastalert/es_mappings/5/elastalert_status.json new file mode 100644 index 000000000..59a93351e --- /dev/null +++ b/elastalert/es_mappings/5/elastalert_status.json @@ -0,0 +1,13 @@ +{ + "elastalert_status": { + "properties": { + "rule_name": { + "type": "keyword" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} \ No newline at end of file diff --git a/elastalert/es_mappings/5/past_elastalert.json b/elastalert/es_mappings/5/past_elastalert.json new file mode 100644 index 000000000..0ad8c2d33 --- /dev/null +++ b/elastalert/es_mappings/5/past_elastalert.json @@ -0,0 +1,20 @@ +{ + "past_elastalert": { + "properties": { + "rule_name": { + "type": "keyword" + }, + "match_body": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "aggregate_id": { + "type": "keyword" + } + } + } +} \ No newline at end of file diff --git a/elastalert/es_mappings/5/silence.json b/elastalert/es_mappings/5/silence.json new file mode 100644 index 000000000..662008bc5 --- /dev/null +++ b/elastalert/es_mappings/5/silence.json @@ -0,0 +1,17 @@ +{ + "silence": { + "properties": { + "rule_name": { + "type": "keyword" + }, + "until": { + "type": "date", + "format": "dateOptionalTime" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} \ No newline at end of file diff --git a/elastalert/es_mappings/6/elastalert.json b/elastalert/es_mappings/6/elastalert.json new file mode 100644 index 000000000..5fbdebc6c --- /dev/null +++ b/elastalert/es_mappings/6/elastalert.json @@ -0,0 +1,30 @@ +{ + "elastalert": { + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "alert_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_body": { + "type": "object", + "enabled": "false" + }, + "aggregate_id": { + "index": "not_analyzed", + "type": "string" + } + } + } +} \ No newline at end of file diff --git a/elastalert/es_mappings/6/elastalert_error.json b/elastalert/es_mappings/6/elastalert_error.json new file mode 100644 index 000000000..8672dbd94 --- /dev/null +++ b/elastalert/es_mappings/6/elastalert_error.json @@ -0,0 +1,14 @@ +{ + "elastalert_error": { + "properties": { + "data": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} \ No newline at end of file diff --git a/elastalert/es_mappings/6/elastalert_status.json b/elastalert/es_mappings/6/elastalert_status.json new file mode 100644 index 000000000..597ea35a8 --- /dev/null +++ b/elastalert/es_mappings/6/elastalert_status.json @@ -0,0 +1,14 @@ +{ + "elastalert_status": { + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} \ No newline at end of file diff --git a/elastalert/es_mappings/6/past_elastalert.json b/elastalert/es_mappings/6/past_elastalert.json new file mode 100644 index 000000000..9fc943a1b --- /dev/null +++ b/elastalert/es_mappings/6/past_elastalert.json @@ -0,0 +1,22 @@ +{ + "past_elastalert": { + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "match_body": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "aggregate_id": { + "index": "not_analyzed", + "type": "string" + } + } + } +} \ No newline at end of file diff --git a/elastalert/es_mappings/6/silence.json b/elastalert/es_mappings/6/silence.json new file mode 100644 index 000000000..9d52c6883 --- /dev/null +++ b/elastalert/es_mappings/6/silence.json @@ -0,0 +1,18 @@ +{ + "silence": { + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "until": { + "type": "date", + "format": "dateOptionalTime" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} \ No newline at end of file From 5d6bc7a3877156769ce7ddf7fb97bbf12215329c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Wed, 27 Mar 2019 14:56:22 +0100 Subject: [PATCH 159/264] Remove mapping types in Elastic 6 index mappings --- elastalert/es_mappings/6/elastalert.json | 50 +++++++++---------- .../es_mappings/6/elastalert_error.json | 18 +++---- .../es_mappings/6/elastalert_status.json | 18 +++---- elastalert/es_mappings/6/past_elastalert.json | 34 ++++++------- elastalert/es_mappings/6/silence.json | 26 +++++----- 5 files changed, 68 insertions(+), 78 deletions(-) diff --git a/elastalert/es_mappings/6/elastalert.json b/elastalert/es_mappings/6/elastalert.json index 5fbdebc6c..d12861184 100644 --- a/elastalert/es_mappings/6/elastalert.json +++ b/elastalert/es_mappings/6/elastalert.json @@ -1,30 +1,28 @@ { - "elastalert": { - "properties": { - "rule_name": { - "index": "not_analyzed", - "type": "string" - }, - "@timestamp": { - "type": "date", - "format": "dateOptionalTime" - }, - "alert_time": { - "type": "date", - "format": "dateOptionalTime" - }, - "match_time": { - "type": "date", - "format": "dateOptionalTime" - }, - "match_body": { - "type": "object", - "enabled": "false" - }, - "aggregate_id": { - "index": "not_analyzed", - "type": "string" - } + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "alert_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_body": { + "type": "object", + "enabled": "false" + }, + "aggregate_id": { + "index": "not_analyzed", + "type": "string" } } } \ No newline at end of file diff --git a/elastalert/es_mappings/6/elastalert_error.json b/elastalert/es_mappings/6/elastalert_error.json index 8672dbd94..c7be1dc13 100644 --- a/elastalert/es_mappings/6/elastalert_error.json +++ b/elastalert/es_mappings/6/elastalert_error.json @@ -1,14 +1,12 @@ { - "elastalert_error": { - "properties": { - "data": { - "type": "object", - "enabled": "false" - }, - "@timestamp": { - "type": "date", - "format": "dateOptionalTime" - } + "properties": { + "data": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" } } } \ No newline at end of file diff --git a/elastalert/es_mappings/6/elastalert_status.json b/elastalert/es_mappings/6/elastalert_status.json index 597ea35a8..2b4f1781c 100644 --- a/elastalert/es_mappings/6/elastalert_status.json +++ b/elastalert/es_mappings/6/elastalert_status.json @@ -1,14 +1,12 @@ { - "elastalert_status": { - "properties": { - "rule_name": { - "index": "not_analyzed", - "type": "string" - }, - "@timestamp": { - "type": "date", - "format": "dateOptionalTime" - } + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" } } } \ No newline at end of file diff --git a/elastalert/es_mappings/6/past_elastalert.json b/elastalert/es_mappings/6/past_elastalert.json index 9fc943a1b..0d1675f43 100644 --- a/elastalert/es_mappings/6/past_elastalert.json +++ b/elastalert/es_mappings/6/past_elastalert.json @@ -1,22 +1,20 @@ { - "past_elastalert": { - "properties": { - "rule_name": { - "index": "not_analyzed", - "type": "string" - }, - "match_body": { - "type": "object", - "enabled": "false" - }, - "@timestamp": { - "type": "date", - "format": "dateOptionalTime" - }, - "aggregate_id": { - "index": "not_analyzed", - "type": "string" - } + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "match_body": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "aggregate_id": { + "index": "not_analyzed", + "type": "string" } } } \ No newline at end of file diff --git a/elastalert/es_mappings/6/silence.json b/elastalert/es_mappings/6/silence.json index 9d52c6883..c917347cb 100644 --- a/elastalert/es_mappings/6/silence.json +++ b/elastalert/es_mappings/6/silence.json @@ -1,18 +1,16 @@ { - "silence": { - "properties": { - "rule_name": { - "index": "not_analyzed", - "type": "string" - }, - "until": { - "type": "date", - "format": "dateOptionalTime" - }, - "@timestamp": { - "type": "date", - "format": "dateOptionalTime" - } + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "until": { + "type": "date", + "format": "dateOptionalTime" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" } } } \ No newline at end of file From fe02d358b58f033b211366b4e707f0ac03cef9ef Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Thu, 28 Mar 2019 09:36:15 +0100 Subject: [PATCH 160/264] Create index using es_mapping files. Use preferred _doc type name for Elasticsearch > 5 so that index APIs have the same path as they will have in 7.0 --- elastalert/create_index.py | 135 ++++++++++--------------------------- tests/create_index_test.py | 53 +++++++++++++++ 2 files changed, 87 insertions(+), 101 deletions(-) create mode 100644 tests/create_index_test.py diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 86baf6ab2..197d6026e 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -6,6 +6,7 @@ import getpass import os import time +import json import elasticsearch.helpers import yaml @@ -16,7 +17,6 @@ from elasticsearch.exceptions import NotFoundError from envparse import Env - env = Env(ES_USE_SSL=bool) @@ -128,93 +128,7 @@ def main(): print("Elastic Version:" + esversion.split(".")[0]) elasticversion = int(esversion.split(".")[0]) - if(elasticversion > 5): - mapping = {'type': 'keyword'} - else: - mapping = {'index': 'not_analyzed', 'type': 'string'} - - print("Mapping used for string:" + str(mapping)) - - silence_mapping = { - 'silence': { - 'properties': { - 'rule_name': mapping, - 'until': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - }, - }, - } - ess_mapping = { - 'elastalert_status': { - 'properties': { - 'rule_name': mapping, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - }, - }, - } - es_mapping = { - 'elastalert': { - 'properties': { - 'rule_name': mapping, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - 'alert_time': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - 'match_time': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - 'match_body': { - 'type': 'object', - 'enabled': False, - }, - 'aggregate_id': mapping, - }, - }, - } - past_mapping = { - 'past_elastalert': { - 'properties': { - 'rule_name': mapping, - 'match_body': { - 'type': 'object', - 'enabled': False, - }, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - 'aggregate_id': mapping, - }, - }, - } - error_mapping = { - 'elastalert_error': { - 'properties': { - 'data': { - 'type': 'object', - 'enabled': False, - }, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - }, - }, - } + es_index_mappings = read_es_index_mappings() if elasticversion > 5 else read_es_index_mappings(5) es_index = IndicesClient(es) if not args.recreate: @@ -248,21 +162,20 @@ def main(): # To avoid a race condition. TODO: replace this with a real check time.sleep(2) - if(elasticversion > 5): - es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) - es.indices.put_mapping(index=index + '_status', doc_type='elastalert_status', body=ess_mapping) - es.indices.put_mapping(index=index + '_silence', doc_type='silence', body=silence_mapping) - es.indices.put_mapping(index=index + '_error', doc_type='elastalert_error', body=error_mapping) - es.indices.put_mapping(index=index + '_past', doc_type='past_elastalert', body=past_mapping) - print('New index %s created' % index) + if elasticversion > 5: + es.indices.put_mapping(index=index, doc_type='_doc', body=es_index_mappings['elastalert']) + es.indices.put_mapping(index=index + '_status', doc_type='_doc', body=es_index_mappings['elastalert_status']) + es.indices.put_mapping(index=index + '_silence', doc_type='_doc', body=es_index_mappings['silence']) + es.indices.put_mapping(index=index + '_error', doc_type='_doc', body=es_index_mappings['elastalert_error']) + es.indices.put_mapping(index=index + '_past', doc_type='_doc', body=es_index_mappings['past_elastalert']) else: - es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) - es.indices.put_mapping(index=index, doc_type='elastalert_status', body=ess_mapping) - es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping) - es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping) - es.indices.put_mapping(index=index, doc_type='past_elastalert', body=past_mapping) - print('New index %s created' % index) + es.indices.put_mapping(index=index, doc_type='elastalert', body=es_index_mappings['elastalert']) + es.indices.put_mapping(index=index, doc_type='elastalert_status', body=es_index_mappings['elastalert_status']) + es.indices.put_mapping(index=index, doc_type='silence', body=es_index_mappings['silence']) + es.indices.put_mapping(index=index, doc_type='elastalert_error', body=es_index_mappings['elastalert_error']) + es.indices.put_mapping(index=index, doc_type='past_elastalert', body=es_index_mappings['past_elastalert']) + print('New index %s created' % index) if old_index: print("Copying all data from old index '{0}' to new index '{1}'".format(old_index, index)) # Use the defaults for chunk_size, scroll, scan_kwargs, and bulk_kwargs @@ -271,5 +184,25 @@ def main(): print('Done!') +def read_es_index_mappings(es_version=6): + print('Reading Elastic {0} index mappings:'.format(es_version)) + return { + 'silence': read_es_index_mapping('silence', es_version), + 'elastalert_status': read_es_index_mapping('elastalert_status', es_version), + 'elastalert': read_es_index_mapping('elastalert', es_version), + 'past_elastalert': read_es_index_mapping('past_elastalert', es_version), + 'elastalert_error': read_es_index_mapping('elastalert_error', es_version) + } + + +def read_es_index_mapping(mapping, es_version=6): + base_path = os.path.abspath(os.path.dirname(__file__)) + mapping_path = 'es_mappings/{0}/{1}.json'.format(es_version, mapping) + path = os.path.join(base_path, mapping_path) + with open(path, 'r') as f: + print("Reading index mapping '{0}'".format(mapping_path)) + return json.load(f) + + if __name__ == '__main__': main() diff --git a/tests/create_index_test.py b/tests/create_index_test.py new file mode 100644 index 000000000..ba306aee5 --- /dev/null +++ b/tests/create_index_test.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +import json + +import pytest + +import elastalert.create_index + +es_mappings = [ + 'elastalert', + 'elastalert_error', + 'elastalert_status', + 'past_elastalert', + 'silence' +] + + +@pytest.mark.parametrize('es_mapping', es_mappings) +def test_read_default_index_mapping(es_mapping): + mapping = elastalert.create_index.read_es_index_mapping(es_mapping) + assert es_mapping not in mapping + print(json.dumps(mapping, indent=2)) + + +@pytest.mark.parametrize('es_mapping', es_mappings) +def test_read_es_5_index_mapping(es_mapping): + mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 5) + assert es_mapping in mapping + print(json.dumps(mapping, indent=2)) + + +@pytest.mark.parametrize('es_mapping', es_mappings) +def test_read_es_6_index_mapping(es_mapping): + mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 6) + assert es_mapping not in mapping + print(json.dumps(mapping, indent=2)) + + +def test_read_default_index_mappings(): + mappings = elastalert.create_index.read_es_index_mappings() + assert len(mappings) == len(es_mappings) + print(json.dumps(mappings, indent=2)) + + +def test_read_es_5_index_mappings(): + mappings = elastalert.create_index.read_es_index_mappings(5) + assert len(mappings) == len(es_mappings) + print(json.dumps(mappings, indent=2)) + + +def test_read_es_6_index_mappings(): + mappings = elastalert.create_index.read_es_index_mappings(6) + assert len(mappings) == len(es_mappings) + print(json.dumps(mappings, indent=2)) From 940f187ebb9b9d5cc9a9a76daadbef25dd5ead2d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Thu, 28 Mar 2019 11:23:19 +0100 Subject: [PATCH 161/264] Add elasticsearch test stage to travis ci --- .travis.yml | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/.travis.yml b/.travis.yml index 3b189d529..d0016f716 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,6 +7,19 @@ env: install: - pip install tox script: make test +jobs: + include: + - stage: "Elasticsearch" + env: + - TOXENV=py27 ES_VERSION=7.0.0-rc1 + install: + - pip install tox + - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz + - tar -xzf elasticsearch-${ES_VERSION}.tar.gz + - ./elasticsearch-${ES_VERSION}/bin/elasticsearch & + before_script: + - sleep 10 + script: make test deploy: provider: pypi user: yelplabs From f900674458482f745edcd7e7bc48877737b1950d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Thu, 28 Mar 2019 12:29:35 +0100 Subject: [PATCH 162/264] Fix broken TravisCI Elasticsearch build stage --- .travis.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.travis.yml b/.travis.yml index d0016f716..eee80678e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -15,11 +15,11 @@ jobs: install: - pip install tox - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz - - tar -xzf elasticsearch-${ES_VERSION}.tar.gz + - tar -xzf elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz - ./elasticsearch-${ES_VERSION}/bin/elasticsearch & - before_script: - - sleep 10 - script: make test + script: + - wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200 + - make test deploy: provider: pypi user: yelplabs From b942cb0e4cccda7905b649d9464bca40650bdc7e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Fri, 29 Mar 2019 15:07:15 +0100 Subject: [PATCH 163/264] Add support for 6.6 <= Elasticsearch <= 7.x --- elastalert/create_index.py | 2 +- elastalert/elastalert.py | 123 +++++++++++++++++++++++---------- elastalert/rule_from_kibana.py | 18 ++++- tests/base_test.py | 123 ++++++++++++++++++++++++++------- tests/conftest.py | 59 ++++++++++++++++ 5 files changed, 260 insertions(+), 65 deletions(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 197d6026e..d12ef0cbd 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -125,7 +125,7 @@ def main(): client_key=client_key) esversion = es.info()["version"]["number"] - print("Elastic Version:" + esversion.split(".")[0]) + print("Elastic Version:" + esversion) elasticversion = int(esversion.split(".")[0]) es_index_mappings = read_es_index_mappings() if elasticversion > 5 else read_es_index_mappings(5) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 22b9d49c8..0d8df5f30 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -161,6 +161,12 @@ def is_atleastfive(self): def is_atleastsix(self): return int(self.es_version.split(".")[0]) >= 6 + def is_atleastsixsix(self): + return float('.'.join(self.es_version.split(".")[:2])) >= 6.6 + + def is_atleastseven(self): + return int(self.es_version.split(".")[0]) >= 7 + @staticmethod def get_index(rule, starttime=None, endtime=None): """ Gets the index for a rule. If strftime is set and starttime and endtime @@ -283,7 +289,13 @@ def get_index_start(self, index, timestamp_field='@timestamp'): """ query = {'sort': {timestamp_field: {'order': 'asc'}}} try: - res = self.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field], ignore_unavailable=True) + if self.is_atleastsixsix(): + # TODO use _source_includes=[...] instead when elasticsearch client supports this + res = self.current_es.search(index=index, size=1, body=query, + params={'_source_includes': timestamp_field}, ignore_unavailable=True) + else: + res = self.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field], + ignore_unavailable=True) except ElasticsearchException as e: self.handle_error("Elasticsearch query error: %s" % (e), {'index': index, 'query': query}) return '1969-12-30T00:00:00Z' @@ -354,7 +366,14 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): to_ts_func=rule['dt_to_ts'], five=rule['five'], ) - extra_args = {'_source_include': rule['include']} + if self.is_atleastsixsix(): + # TODO fix when elasticsearch client supports param _source_includes + # Since _source_includes is not supported we must use params instead. + # the value object in _source_includes is not automagically parsed into a legal + # url query parameter array so we must explicitly handle that as well + extra_args = {'params': {'_source_includes': (','.join(rule['include']))}} + else: + extra_args = {'_source_include': rule['include']} scroll_keepalive = rule.get('scroll_keepalive', self.scroll_keepalive) if not rule.get('_source_enabled'): if rule['five']: @@ -375,7 +394,11 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): ignore_unavailable=True, **extra_args ) - self.total_hits = int(res['hits']['total']) + + if self.is_atleastseven(): + self.total_hits = int(res['hits']['total']['value']) + else: + self.total_hits = int(res['hits']['total']) if len(res.get('_shards', {}).get('failures', [])) > 0: try: @@ -501,7 +524,8 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule['doc_type'], body=query, size=0, ignore_unavailable=True) + res = self.current_es.search(index=index, doc_type=rule['doc_type'], body=query, size=0, + ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -547,7 +571,8 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule.get('doc_type'), body=query, size=0, ignore_unavailable=True) + res = self.current_es.search(index=index, doc_type=rule.get('doc_type'), body=query, size=0, + ignore_unavailable=True) except ElasticsearchException as e: if len(str(e)) > 1024: e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024) @@ -559,7 +584,12 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ payload = res['aggregations']['filtered'] else: payload = res['aggregations'] - self.num_hits += res['hits']['total'] + + if self.is_atleastseven(): + self.num_hits += res['hits']['total']['value'] + else: + self.num_hits += res['hits']['total'] + return {endtime: payload} def remove_duplicate_events(self, data, rule): @@ -655,8 +685,13 @@ def get_starttime(self, rule): try: if self.is_atleastsix(): index = self.get_six_index('elastalert_status') - res = self.writeback_es.search(index=index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) + if self.is_atleastsixsix(): + # TODO use _source_includes=[...] instead when elasticsearch client supports this + res = self.writeback_es.search(index=index, doc_type='_doc', size=1, body=query, + params={'_source_includes': 'endtime,rule_name'}) + else: + res = self.writeback_es.search(index=index, doc_type='_doc', size=1, body=query, + _source_include=['endtime', 'rule_name']) else: res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert_status', size=1, body=query, _source_include=['endtime', 'rule_name']) @@ -1281,7 +1316,7 @@ def upload_dashboard(self, db, rule, match): # Upload es = elasticsearch_client(rule) - + # TODO: doc_type = _doc for elastic >= 6 res = es.index(index='kibana-int', doc_type='temp', body=db_body) @@ -1300,7 +1335,13 @@ def get_dashboard(self, rule, db_name): raise EAException("use_kibana_dashboard undefined") query = {'query': {'term': {'_id': db_name}}} try: - res = es.search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) + if self.is_atleastsixsix(): + # TODO use doc_type = _doc + # TODO use _source_includes=[...] instead when elasticsearch client supports for this + res = es.search(index='kibana-int', doc_type='dashboard', body=query, + params={'_source_includes': 'dashboard'}) + else: + res = es.search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) except ElasticsearchException as e: raise EAException("Error querying for dashboard: %s" % (e)), None, sys.exc_info()[2] @@ -1473,10 +1514,6 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No return body def writeback(self, doc_type, body): - writeback_index = self.writeback_index - if(self.is_atleastsix()): - writeback_index = self.get_six_index(doc_type) - # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: writeback_body = replace_dots_in_field_names(body) @@ -1496,8 +1533,11 @@ def writeback(self, doc_type, body): writeback_body['@timestamp'] = dt_to_ts(ts_now()) try: - res = self.writeback_es.index(index=writeback_index, - doc_type=doc_type, body=body) + if self.is_atleastsix(): + writeback_index = self.get_six_index(doc_type) + res = self.writeback_es.index(index=writeback_index, doc_type='_doc', body=body) + else: + res = self.writeback_es.index(index=self.writeback_index, doc_type=doc_type, body=body) return res except ElasticsearchException as e: logging.exception("Error writing alert info to Elasticsearch: %s" % (e)) @@ -1520,10 +1560,10 @@ def find_recent_pending_alerts(self, time_limit): query = {'query': inner_query, 'filter': time_filter} query.update(sort) try: - res = self.writeback_es.search(index=self.writeback_index, - doc_type='elastalert', - body=query, - size=1000) + if self.is_atleastsix(): + res = self.writeback_es.search(index=self.writeback_index, doc_type='_doc', body=query, size=1000) + else: + res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert', body=query, size=1000) if res['hits']['hits']: return res['hits']['hits'] except ElasticsearchException as e: @@ -1576,9 +1616,10 @@ def send_pending_alerts(self): # Delete it from the index try: - self.writeback_es.delete(index=self.writeback_index, - doc_type='elastalert', - id=_id) + if self.is_atleastsix(): + self.writeback_es.delete(index=self.writeback_index, doc_type='_doc', id=_id) + else: + self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=_id) except ElasticsearchException: # TODO: Give this a more relevant exception, try:except: is evil. self.handle_error("Failed to delete alert %s at %s" % (_id, alert_time)) @@ -1608,15 +1649,18 @@ def get_aggregated_matches(self, _id): query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}} matches = [] try: - res = self.writeback_es.search(index=self.writeback_index, - doc_type='elastalert', - body=query, - size=self.max_aggregation) + if self.is_atleastsix(): + res = self.writeback_es.search(index=self.writeback_index, doc_type='_doc', body=query, + size=self.max_aggregation) + else: + res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert', body=query, + size=self.max_aggregation) for match in res['hits']['hits']: matches.append(match['_source']) - self.writeback_es.delete(index=self.writeback_index, - doc_type='elastalert', - id=match['_id']) + if self.is_atleastsix(): + self.writeback_es.delete(index=self.writeback_index, doc_type='_doc', id=match['_id']) + else: + self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=match['_id']) except (KeyError, ElasticsearchException) as e: self.handle_error("Error fetching aggregated matches: %s" % (e), {'id': _id}) return matches @@ -1632,10 +1676,10 @@ def find_pending_aggregate_alert(self, rule, aggregation_key_value=None): query = {'query': {'bool': query}} query['sort'] = {'alert_time': {'order': 'desc'}} try: - res = self.writeback_es.search(index=self.writeback_index, - doc_type='elastalert', - body=query, - size=1) + if self.is_atleastsix(): + res = self.writeback_es.search(index=self.writeback_index, doc_type='_doc', body=query, size=1) + else: + res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert', body=query, size=1) if len(res['hits']['hits']) == 0: return None except (KeyError, ElasticsearchException) as e: @@ -1774,10 +1818,15 @@ def is_silenced(self, rule_name): query.update(sort) try: - if(self.is_atleastsix()): + if self.is_atleastsix(): index = self.get_six_index('silence') - res = self.writeback_es.search(index=index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) + if self.is_atleastsixsix(): + # TODO use _source_includes=[...] instead when elasticsearch client supports this + res = self.writeback_es.search(index=index, doc_type='_doc', + size=1, body=query, params={'_source_includes': 'until,exponent'}) + else: + res = self.writeback_es.search(index=index, doc_type='_doc', + size=1, body=query, _source_include=['until', 'exponent']) else: res = self.writeback_es.search(index=self.writeback_index, doc_type='silence', size=1, body=query, _source_include=['until', 'exponent']) diff --git a/elastalert/rule_from_kibana.py b/elastalert/rule_from_kibana.py index 7589d99d6..a16d150da 100644 --- a/elastalert/rule_from_kibana.py +++ b/elastalert/rule_from_kibana.py @@ -17,8 +17,20 @@ def main(): db_name = raw_input("Dashboard name: ") send_get_body_as = raw_input("Method for querying Elasticsearch[GET]: ") or 'GET' es = Elasticsearch(host=es_host, port=es_port, send_get_body_as=send_get_body_as) + + es_version = es.info()["version"]["number"] + print("Elastic Version:" + es_version) + query = {'query': {'term': {'_id': db_name}}} - res = es.search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) + + if is_atleastsixsix(es_version): + # TODO add support for kibana 5 + # TODO use doc_type='_doc' instead + # TODO use _source_includes=[...] instead when elasticsearch client supports this + res = es.search(index='kibana-int', doc_type='dashboard', body=query, params={'_source_includes': 'dashboard'}) + else: + res = es.search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) + if not res['hits']['hits']: print("No dashboard %s found" % (db_name)) exit() @@ -35,5 +47,9 @@ def main(): print(yaml.safe_dump(config_filters)) +def is_atleastsixsix(es_version): + return float('.'.join(es_version.split('.')[:2])) >= 6.6 + + if __name__ == '__main__': main() diff --git a/tests/base_test.py b/tests/base_test.py index 4aa2b0cfc..efd986d10 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -22,7 +22,6 @@ from elastalert.util import ts_to_dt from elastalert.util import unix_to_dt - START_TIMESTAMP = '2014-09-26T12:34:45Z' END_TIMESTAMP = '2014-09-27T12:34:45Z' START = ts_to_dt(START_TIMESTAMP) @@ -91,21 +90,47 @@ def test_query(ea): ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) ea.current_es.search.assert_called_with(body={ - 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, + 'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], + ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) +def test_query_sixsix(ea_sixsix): + ea_sixsix.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.run_query(ea_sixsix.rules[0], START, END) + ea_sixsix.current_es.search.assert_called_with(body={ + 'query': {'bool': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', params={'_source_includes': '@timestamp'}, + ignore_unavailable=True, + size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) + + def test_query_with_fields(ea): ea.rules[0]['_source_enabled'] = False ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) ea.current_es.search.assert_called_with(body={ - 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, + 'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}], 'fields': ['@timestamp']}, index='idx', ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) +def test_query_sixsix_with_fields(ea_sixsix): + ea_sixsix.rules[0]['_source_enabled'] = False + ea_sixsix.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.run_query(ea_sixsix.rules[0], START, END) + ea_sixsix.current_es.search.assert_called_with(body={ + 'query': {'bool': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}], 'stored_fields': ['@timestamp']}, index='idx', + ignore_unavailable=True, + size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) + + def test_query_with_unix(ea): ea.rules[0]['timestamp_type'] = 'unix' ea.rules[0]['dt_to_ts'] = dt_to_unix @@ -114,11 +139,28 @@ def test_query_with_unix(ea): start_unix = dt_to_unix(START) end_unix = dt_to_unix(END) ea.current_es.search.assert_called_with( - body={'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, + body={'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], + ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) +def test_query_sixsix_with_unix(ea_sixsix): + ea_sixsix.rules[0]['timestamp_type'] = 'unix' + ea_sixsix.rules[0]['dt_to_ts'] = dt_to_unix + ea_sixsix.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.run_query(ea_sixsix.rules[0], START, END) + start_unix = dt_to_unix(START) + end_unix = dt_to_unix(END) + ea_sixsix.current_es.search.assert_called_with( + body={'query': {'bool': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', params={'_source_includes': '@timestamp'}, + ignore_unavailable=True, + size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) + + def test_query_with_unixms(ea): ea.rules[0]['timestamp_type'] = 'unixms' ea.rules[0]['dt_to_ts'] = dt_to_unixms @@ -127,11 +169,28 @@ def test_query_with_unixms(ea): start_unix = dt_to_unixms(START) end_unix = dt_to_unixms(END) ea.current_es.search.assert_called_with( - body={'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, + body={'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], + ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) +def test_query_sixsix_with_unixms(ea_sixsix): + ea_sixsix.rules[0]['timestamp_type'] = 'unixms' + ea_sixsix.rules[0]['dt_to_ts'] = dt_to_unixms + ea_sixsix.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.run_query(ea_sixsix.rules[0], START, END) + start_unix = dt_to_unixms(START) + end_unix = dt_to_unixms(END) + ea_sixsix.current_es.search.assert_called_with( + body={'query': {'bool': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', params={'_source_includes': '@timestamp'}, + ignore_unavailable=True, + size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) + + def test_no_hits(ea): ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) @@ -409,7 +468,8 @@ def test_agg_cron(ea): with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch('elastalert.elastalert.croniter.get_next') as mock_ts: # Aggregate first two, query over full range - mock_ts.side_effect = [dt_to_unix(ts_to_dt('2014-09-26T12:46:00')), dt_to_unix(ts_to_dt('2014-09-26T13:04:00'))] + mock_ts.side_effect = [dt_to_unix(ts_to_dt('2014-09-26T12:46:00')), + dt_to_unix(ts_to_dt('2014-09-26T13:04:00'))] ea.rules[0]['aggregation'] = {'schedule': '*/5 * * * *'} ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps] ea.run_rule(ea.rules[0], END, START) @@ -697,7 +757,8 @@ def test_count(ea): # Assert that es.count is run against every run_every timeframe between START and END start = START query = { - 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}} + 'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}} while END - start > ea.run_every: end = start + ea.run_every query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end) @@ -718,7 +779,8 @@ def run_and_assert_segmented_queries(ea, start, end, segment_size): # Assert elastalert_status was created for the entire time range assert ea.writeback_es.index.call_args_list[-1][1]['body']['starttime'] == dt_to_ts(original_start) if ea.rules[0].get('aggregation_query_element'): - assert ea.writeback_es.index.call_args_list[-1][1]['body']['endtime'] == dt_to_ts(original_end - (original_end - end)) + assert ea.writeback_es.index.call_args_list[-1][1]['body']['endtime'] == dt_to_ts( + original_end - (original_end - end)) assert original_end - end < segment_size else: assert ea.writeback_es.index.call_args_list[-1][1]['body']['endtime'] == dt_to_ts(original_end) @@ -729,6 +791,7 @@ def test_query_segmenting_reset_num_hits(ea): def assert_num_hits_reset(): assert ea.num_hits == 0 ea.num_hits += 10 + with mock.patch.object(ea, 'run_query') as mock_run_query: mock_run_query.side_effect = assert_num_hits_reset() ea.run_rule(ea.rules[0], END, START) @@ -914,7 +977,7 @@ def test_kibana_dashboard(ea): found_filters = 0 for filter_id, filter_dict in db['services']['filter']['list'].items(): if (filter_dict['field'] == 'foo' and filter_dict['query'] == '"cat"') or \ - (filter_dict['field'] == 'bar' and filter_dict['query'] == '"dog"'): + (filter_dict['field'] == 'bar' and filter_dict['query'] == '"dog"'): found_filters += 1 continue assert found_filters == 2 @@ -954,7 +1017,8 @@ def test_rule_changes(ea): with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: with mock.patch('elastalert.elastalert.load_configuration') as mock_load: with mock.patch.object(ea, 'send_notification_email') as mock_send: - mock_load.return_value = {'filter': [], 'name': 'rule3', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} + mock_load.return_value = {'filter': [], 'name': 'rule3', 'new': 'stuff', + 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes ea.load_rule_changes() mock_send.assert_called_once_with(exception=mock.ANY, rule_file='rules/rule4.yaml') @@ -966,7 +1030,8 @@ def test_rule_changes(ea): new_hashes.update({'rules/rule4.yaml': 'asdf'}) with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: with mock.patch('elastalert.elastalert.load_configuration') as mock_load: - mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, 'rule_file': 'rules/rule4.yaml'} + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, + 'rule_file': 'rules/rule4.yaml'} mock_hashes.return_value = new_hashes ea.load_rule_changes() assert len(ea.rules) == 3 @@ -1008,13 +1073,16 @@ def test_count_keys(ea): ea.rules[0]['top_count_keys'] = ['this', 'that'] ea.rules[0]['type'].matches = {'@timestamp': END} ea.rules[0]['doc_type'] = 'blah' - buckets = [{'aggregations': {'filtered': {'counts': {'buckets': [{'key': 'a', 'doc_count': 10}, {'key': 'b', 'doc_count': 5}]}}}}, - {'aggregations': {'filtered': {'counts': {'buckets': [{'key': 'd', 'doc_count': 10}, {'key': 'c', 'doc_count': 12}]}}}}] + buckets = [{'aggregations': { + 'filtered': {'counts': {'buckets': [{'key': 'a', 'doc_count': 10}, {'key': 'b', 'doc_count': 5}]}}}}, + {'aggregations': {'filtered': { + 'counts': {'buckets': [{'key': 'd', 'doc_count': 10}, {'key': 'c', 'doc_count': 12}]}}}}] ea.current_es.search.side_effect = buckets counts = ea.get_top_counts(ea.rules[0], START, END, ['this', 'that']) calls = ea.current_es.search.call_args_list assert calls[0][1]['search_type'] == 'count' - assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5, 'min_doc_count': 1} + assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5, + 'min_doc_count': 1} assert counts['top_events_this'] == {'a': 10, 'b': 5} assert counts['top_events_that'] == {'d': 10, 'c': 12} @@ -1030,13 +1098,13 @@ def test_exponential_realert(ea): ts5m = until + datetime.timedelta(minutes=5) ts4h = until + datetime.timedelta(hours=4) - test_values = [(ts5s, until, 0), # Exp will increase to 1, 10*2**0 = 10s + test_values = [(ts5s, until, 0), # Exp will increase to 1, 10*2**0 = 10s (ts15s, until, 0), # Exp will stay at 0, 10*2**0 = 10s (ts15s, until, 1), # Exp will increase to 2, 10*2**1 = 20s - (ts1m, until, 2), # Exp will decrease to 1, 10*2**2 = 40s - (ts1m, until, 3), # Exp will increase to 4, 10*2**3 = 1m20s - (ts5m, until, 1), # Exp will lower back to 0, 10*2**1 = 20s - (ts4h, until, 9), # Exp will lower back to 0, 10*2**9 = 1h25m + (ts1m, until, 2), # Exp will decrease to 1, 10*2**2 = 40s + (ts1m, until, 3), # Exp will increase to 4, 10*2**3 = 1m20s + (ts5m, until, 1), # Exp will lower back to 0, 10*2**1 = 20s + (ts4h, until, 9), # Exp will lower back to 0, 10*2**9 = 1h25m (ts4h, until, 10), # Exp will lower back to 9, 10*2**10 = 2h50m (ts4h, until, 11)] # Exp will increase to 12, 10*2**11 = 5h results = (1, 0, 2, 1, 4, 0, 0, 9, 12) @@ -1053,7 +1121,7 @@ def test_wait_until_responsive(ea): # Takes a while before becoming responsive. ea.writeback_es.indices.exists.side_effect = [ ConnectionError(), # ES is not yet responsive. - False, # index does not yet exist. + False, # index does not yet exist. True, ] @@ -1264,7 +1332,8 @@ def test_query_with_whitelist_filter_es_five(ea): ea.rules[0]['whitelist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] new_rule = copy.copy(ea.rules[0]) ea.init_rule(new_rule, True) - assert 'NOT username:"xudan1" AND NOT username:"xudan12" AND NOT username:"aa1"' in new_rule['filter'][-1]['query_string']['query'] + assert 'NOT username:"xudan1" AND NOT username:"xudan12" AND NOT username:"aa1"' in \ + new_rule['filter'][-1]['query_string']['query'] def test_query_with_blacklist_filter_es(ea): @@ -1274,7 +1343,8 @@ def test_query_with_blacklist_filter_es(ea): ea.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] new_rule = copy.copy(ea.rules[0]) ea.init_rule(new_rule, True) - assert 'username:"xudan1" OR username:"xudan12" OR username:"aa1"' in new_rule['filter'][-1]['query']['query_string']['query'] + assert 'username:"xudan1" OR username:"xudan12" OR username:"aa1"' in \ + new_rule['filter'][-1]['query']['query_string']['query'] def test_query_with_blacklist_filter_es_five(ea): @@ -1285,4 +1355,5 @@ def test_query_with_blacklist_filter_es_five(ea): ea.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] new_rule = copy.copy(ea.rules[0]) ea.init_rule(new_rule, True) - assert 'username:"xudan1" OR username:"xudan12" OR username:"aa1"' in new_rule['filter'][-1]['query_string']['query'] + assert 'username:"xudan1" OR username:"xudan12" OR username:"aa1"' in new_rule['filter'][-1]['query_string'][ + 'query'] diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..d585b1c19 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -13,6 +13,7 @@ mock_info = {'status': 200, 'name': 'foo', 'version': {'number': '2.0'}} +mock_sixsix_info = {'status': 200, 'name': 'foo', 'version': {'number': '6.6.0'}} @pytest.fixture(scope='function', autouse=True) @@ -51,6 +52,20 @@ def __init__(self, host='es', port=14900): self.indices = mock_es_indices_client() +class mock_es_sixsix_client(object): + def __init__(self, host='es', port=14900): + self.host = host + self.port = port + self.return_hits = [] + self.search = mock.Mock() + self.create = mock.Mock() + self.index = mock.Mock() + self.delete = mock.Mock() + self.info = mock.Mock(return_value=mock_sixsix_info) + self.ping = mock.Mock(return_value=True) + self.indices = mock_es_indices_client() + + class mock_ruletype(object): def __init__(self): self.add_data = mock.Mock() @@ -114,6 +129,50 @@ def ea(): return ea +@pytest.fixture +def ea_sixsix(): + rules = [{'es_host': '', + 'es_port': 14900, + 'name': 'anytest', + 'index': 'idx', + 'filter': [], + 'include': ['@timestamp'], + 'aggregation': datetime.timedelta(0), + 'realert': datetime.timedelta(0), + 'processed_hits': {}, + 'timestamp_field': '@timestamp', + 'match_enhancements': [], + 'rule_file': 'blah.yaml', + 'max_query_size': 10000, + 'ts_to_dt': ts_to_dt, + 'dt_to_ts': dt_to_ts, + '_source_enabled': True}] + conf = {'rules_folder': 'rules', + 'run_every': datetime.timedelta(minutes=10), + 'buffer_time': datetime.timedelta(minutes=5), + 'alert_time_limit': datetime.timedelta(hours=24), + 'es_host': 'es', + 'es_port': 14900, + 'writeback_index': 'wb', + 'rules': rules, + 'max_query_size': 10000, + 'old_query_limit': datetime.timedelta(weeks=1), + 'disable_rules_on_error': False, + 'scroll_keepalive': '30s'} + elastalert.elastalert.elasticsearch_client = mock_es_sixsix_client + with mock.patch('elastalert.elastalert.get_rule_hashes'): + with mock.patch('elastalert.elastalert.load_rules') as load_conf: + load_conf.return_value = conf + ea_sixsix = elastalert.elastalert.ElastAlerter(['--pin_rules']) + ea_sixsix.rules[0]['type'] = mock_ruletype() + ea_sixsix.rules[0]['alert'] = [mock_alert()] + ea_sixsix.writeback_es = mock_es_sixsix_client() + ea_sixsix.writeback_es.search.return_value = {'hits': {'hits': []}} + ea_sixsix.writeback_es.index.return_value = {'_id': 'ABCD'} + ea_sixsix.current_es = mock_es_sixsix_client('', -1) + return ea_sixsix + + @pytest.fixture(scope='function') def environ(): """py.test fixture to get a fresh mutable environment.""" From 98c7867cb3ea30879aa3540904c5945021ef8736 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 29 Mar 2019 11:55:15 -0700 Subject: [PATCH 164/264] Version 0.1.39 --- changelog.md | 17 +++++++++++++++++ setup.py | 2 +- 2 files changed, 18 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index d007a4dee..30b52f601 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,22 @@ # Change Log +# v0.1.39 + +### Added +- Added spike alerts for metric aggregations +- Allow SSL connections for Stomp +- Allow limits on alert text length +- Add optional min doc count for terms queries +- Add ability to index into arrays for alert_text_args, etc + +### Fixed +- Fixed bug involving --config flag with create-index +- Fixed some settings not being inherited from the config properly +- Some fixes for Hive alerter +- Close SMTP connections properly +- Fix timestamps in Pagerduty v2 payload +- Fixed an bug causing aggregated alerts to mix up + # v0.1.38 ### Added diff --git a/setup.py b/setup.py index 91a514f18..ac680c4a9 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.38', + version='0.1.39', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From c0054e27e6b480ef28e0049766d70497da780ad3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Mon, 1 Apr 2019 14:05:36 +0200 Subject: [PATCH 165/264] Test create_index in elasticsearch with bugfixes elasticsearch: - 5.6.16 - 6.3.2 - 6.6.2 - 7.0.0-rc1 --- .travis.yml | 17 +- Makefile | 3 + elastalert/create_index.py | 189 ++++++++++-------- elastalert/es_mappings/5/elastalert.json | 6 +- .../es_mappings/5/elastalert_status.json | 3 +- elastalert/es_mappings/5/past_elastalert.json | 6 +- elastalert/es_mappings/5/silence.json | 3 +- elastalert/es_mappings/6/elastalert.json | 6 +- .../es_mappings/6/elastalert_status.json | 3 +- elastalert/es_mappings/6/past_elastalert.json | 6 +- elastalert/es_mappings/6/silence.json | 3 +- elastalert/rule_from_kibana.py | 2 +- pytest.ini | 3 + tests/conftest.py | 17 +- tests/create_index_test.py | 27 +++ 15 files changed, 183 insertions(+), 111 deletions(-) create mode 100644 pytest.ini diff --git a/.travis.yml b/.travis.yml index eee80678e..1f5632943 100644 --- a/.travis.yml +++ b/.travis.yml @@ -10,16 +10,23 @@ script: make test jobs: include: - stage: "Elasticsearch" - env: - - TOXENV=py27 ES_VERSION=7.0.0-rc1 install: - pip install tox - - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz - - tar -xzf elasticsearch-${ES_VERSION}-linux-x86_64.tar.gz + - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz + - mkdir elasticsearch-${ES_VERSION} && tar -xzf elasticsearch-${ES_VERSION}.tar.gz -C elasticsearch-${ES_VERSION} --strip-components=1 - ./elasticsearch-${ES_VERSION}/bin/elasticsearch & script: - wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200 - - make test + - make test-elasticsearch + name: "Elasticsearch 7" + env: TOXENV=py27 ES_VERSION=7.0.0-rc1-linux-x86_64 + - name: "Elasticsearch 6.6" + env: TOXENV=py27 ES_VERSION=6.6.2 + - name: "Elasticsearch 6.3" + env: TOXENV=py27 ES_VERSION=6.3.2 + - name: "Elasticsearch 5.6" + env: TOXENV=py27 ES_VERSION=5.6.16 + deploy: provider: pypi user: yelplabs diff --git a/Makefile b/Makefile index 69f590a48..470062ce8 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ install-hooks: test: tox +test-elasticsearch: + tox -- --runelasticsearch + test-docker: docker-compose --project-name elastalert build tox docker-compose --project-name elastalert run tox diff --git a/elastalert/create_index.py b/elastalert/create_index.py index d12ef0cbd..05296f82e 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -20,7 +20,101 @@ env = Env(ES_USE_SSL=bool) -def main(): +def main(es_client, ea_index, recreate=False, old_ea_index=None): + esversion = es_client.info()["version"]["number"] + print("Elastic Version: " + esversion) + elasticversion = int(esversion.split(".")[0]) + + es_index_mappings = read_es_index_mappings() if elasticversion > 5 else read_es_index_mappings(5) + + es_index = IndicesClient(es_client) + if not recreate: + if es_index.exists(ea_index): + print('Index ' + ea_index + ' already exists. Skipping index creation.') + return None + + # (Re-)Create indices. + if (elasticversion > 5): + index_names = ( + ea_index, + ea_index + '_status', + ea_index + '_silence', + ea_index + '_error', + ea_index + '_past', + ) + else: + index_names = ( + ea_index, + ) + for index_name in index_names: + if es_index.exists(index_name): + print('Deleting index ' + index_name + '.') + try: + es_index.delete(index_name) + except NotFoundError: + # Why does this ever occur?? It shouldn't. But it does. + pass + es_index.create(index_name) + + # To avoid a race condition. TODO: replace this with a real check + time.sleep(2) + + if elasticversion > 5: + # TODO remove doc_type for elasticsearch >= 7 when elastic client supports doc_type=None + params = {'include_type_name': 'true'} if elasticversion > 6 else {} + + es_client.indices.put_mapping(index=ea_index, doc_type='_doc', + body=es_index_mappings['elastalert'], params=params) + es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc', + body=es_index_mappings['elastalert_status'], params=params) + es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc', + body=es_index_mappings['silence'], params=params) + es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc', + body=es_index_mappings['elastalert_error'], params=params) + es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc', + body=es_index_mappings['past_elastalert'], params=params) + else: + es_client.indices.put_mapping(index=ea_index, doc_type='elastalert', + body=es_index_mappings['elastalert']) + es_client.indices.put_mapping(index=ea_index, doc_type='elastalert_status', + body=es_index_mappings['elastalert_status']) + es_client.indices.put_mapping(index=ea_index, doc_type='silence', + body=es_index_mappings['silence']) + es_client.indices.put_mapping(index=ea_index, doc_type='elastalert_error', + body=es_index_mappings['elastalert_error']) + es_client.indices.put_mapping(index=ea_index, doc_type='past_elastalert', + body=es_index_mappings['past_elastalert']) + + print('New index %s created' % ea_index) + if old_ea_index: + print("Copying all data from old index '{0}' to new index '{1}'".format(old_ea_index, ea_index)) + # Use the defaults for chunk_size, scroll, scan_kwargs, and bulk_kwargs + elasticsearch.helpers.reindex(es_client, old_ea_index, ea_index) + + print('Done!') + + +def read_es_index_mappings(es_version=6): + print('Reading Elastic {0} index mappings:'.format(es_version)) + return { + 'silence': read_es_index_mapping('silence', es_version), + 'elastalert_status': read_es_index_mapping('elastalert_status', es_version), + 'elastalert': read_es_index_mapping('elastalert', es_version), + 'past_elastalert': read_es_index_mapping('past_elastalert', es_version), + 'elastalert_error': read_es_index_mapping('elastalert_error', es_version) + } + + +def read_es_index_mapping(mapping, es_version=6): + base_path = os.path.abspath(os.path.dirname(__file__)) + mapping_path = 'es_mappings/{0}/{1}.json'.format(es_version, mapping) + path = os.path.join(base_path, mapping_path) + with open(path, 'r') as f: + print("Reading index mapping '{0}'".format(mapping_path)) + return json.load(f) + + +if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--host', default=os.environ.get('ES_HOST', None), help='Elasticsearch host') parser.add_argument('--port', default=os.environ.get('ES_PORT', None), type=int, help='Elasticsearch port') @@ -31,10 +125,12 @@ def main(): parser.add_argument('--ssl', action='store_true', default=env('ES_USE_SSL', None), help='Use TLS') parser.add_argument('--no-ssl', dest='ssl', action='store_false', help='Do not use TLS') parser.add_argument('--verify-certs', action='store_true', default=None, help='Verify TLS certificates') - parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false', help='Do not verify TLS certificates') + parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false', + help='Do not verify TLS certificates') parser.add_argument('--index', help='Index name to create') parser.add_argument('--old-index', help='Old index name to copy') - parser.add_argument('--send_get_body_as', default='GET', help='Method for querying Elasticsearch - POST, GET or source') + parser.add_argument('--send_get_body_as', default='GET', + help='Method for querying Elasticsearch - POST, GET or source') parser.add_argument( '--boto-profile', default=None, @@ -50,7 +146,8 @@ def main(): help='AWS Region to use for signing requests. Optionally use the AWS_DEFAULT_REGION environment variable') parser.add_argument('--timeout', default=60, help='Elasticsearch request timeout') parser.add_argument('--config', default='config.yaml', help='Global config file (default: config.yaml)') - parser.add_argument('--recreate', type=bool, default=False, help='Force re-creation of the index (this will cause data loss).') + parser.add_argument('--recreate', type=bool, default=False, + help='Force re-creation of the index (this will cause data loss).') args = parser.parse_args() if os.path.isfile(args.config): @@ -104,6 +201,7 @@ def main(): else raw_input('Name of existing index to copy? (Default None) ')) timeout = args.timeout + auth = Auth() http_auth = auth(host=host, username=username, @@ -124,85 +222,4 @@ def main(): ca_certs=ca_certs, client_key=client_key) - esversion = es.info()["version"]["number"] - print("Elastic Version:" + esversion) - elasticversion = int(esversion.split(".")[0]) - - es_index_mappings = read_es_index_mappings() if elasticversion > 5 else read_es_index_mappings(5) - - es_index = IndicesClient(es) - if not args.recreate: - if es_index.exists(index): - print('Index ' + index + ' already exists. Skipping index creation.') - return None - - # (Re-)Create indices. - if (elasticversion > 5): - index_names = ( - index, - index + '_status', - index + '_silence', - index + '_error', - index + '_past', - ) - else: - index_names = ( - index, - ) - for index_name in index_names: - if es_index.exists(index_name): - print('Deleting index ' + index_name + '.') - try: - es_index.delete(index_name) - except NotFoundError: - # Why does this ever occur?? It shouldn't. But it does. - pass - es_index.create(index_name) - - # To avoid a race condition. TODO: replace this with a real check - time.sleep(2) - - if elasticversion > 5: - es.indices.put_mapping(index=index, doc_type='_doc', body=es_index_mappings['elastalert']) - es.indices.put_mapping(index=index + '_status', doc_type='_doc', body=es_index_mappings['elastalert_status']) - es.indices.put_mapping(index=index + '_silence', doc_type='_doc', body=es_index_mappings['silence']) - es.indices.put_mapping(index=index + '_error', doc_type='_doc', body=es_index_mappings['elastalert_error']) - es.indices.put_mapping(index=index + '_past', doc_type='_doc', body=es_index_mappings['past_elastalert']) - else: - es.indices.put_mapping(index=index, doc_type='elastalert', body=es_index_mappings['elastalert']) - es.indices.put_mapping(index=index, doc_type='elastalert_status', body=es_index_mappings['elastalert_status']) - es.indices.put_mapping(index=index, doc_type='silence', body=es_index_mappings['silence']) - es.indices.put_mapping(index=index, doc_type='elastalert_error', body=es_index_mappings['elastalert_error']) - es.indices.put_mapping(index=index, doc_type='past_elastalert', body=es_index_mappings['past_elastalert']) - - print('New index %s created' % index) - if old_index: - print("Copying all data from old index '{0}' to new index '{1}'".format(old_index, index)) - # Use the defaults for chunk_size, scroll, scan_kwargs, and bulk_kwargs - elasticsearch.helpers.reindex(es, old_index, index) - - print('Done!') - - -def read_es_index_mappings(es_version=6): - print('Reading Elastic {0} index mappings:'.format(es_version)) - return { - 'silence': read_es_index_mapping('silence', es_version), - 'elastalert_status': read_es_index_mapping('elastalert_status', es_version), - 'elastalert': read_es_index_mapping('elastalert', es_version), - 'past_elastalert': read_es_index_mapping('past_elastalert', es_version), - 'elastalert_error': read_es_index_mapping('elastalert_error', es_version) - } - - -def read_es_index_mapping(mapping, es_version=6): - base_path = os.path.abspath(os.path.dirname(__file__)) - mapping_path = 'es_mappings/{0}/{1}.json'.format(es_version, mapping) - path = os.path.join(base_path, mapping_path) - with open(path, 'r') as f: - print("Reading index mapping '{0}'".format(mapping_path)) - return json.load(f) - - -if __name__ == '__main__': - main() + main(es_client=es, ea_index=index, recreate=args.recreate, old_ea_index=old_index) diff --git a/elastalert/es_mappings/5/elastalert.json b/elastalert/es_mappings/5/elastalert.json index 989769459..5fbdebc6c 100644 --- a/elastalert/es_mappings/5/elastalert.json +++ b/elastalert/es_mappings/5/elastalert.json @@ -2,7 +2,8 @@ "elastalert": { "properties": { "rule_name": { - "type": "keyword" + "index": "not_analyzed", + "type": "string" }, "@timestamp": { "type": "date", @@ -21,7 +22,8 @@ "enabled": "false" }, "aggregate_id": { - "type": "keyword" + "index": "not_analyzed", + "type": "string" } } } diff --git a/elastalert/es_mappings/5/elastalert_status.json b/elastalert/es_mappings/5/elastalert_status.json index 59a93351e..597ea35a8 100644 --- a/elastalert/es_mappings/5/elastalert_status.json +++ b/elastalert/es_mappings/5/elastalert_status.json @@ -2,7 +2,8 @@ "elastalert_status": { "properties": { "rule_name": { - "type": "keyword" + "index": "not_analyzed", + "type": "string" }, "@timestamp": { "type": "date", diff --git a/elastalert/es_mappings/5/past_elastalert.json b/elastalert/es_mappings/5/past_elastalert.json index 0ad8c2d33..9fc943a1b 100644 --- a/elastalert/es_mappings/5/past_elastalert.json +++ b/elastalert/es_mappings/5/past_elastalert.json @@ -2,7 +2,8 @@ "past_elastalert": { "properties": { "rule_name": { - "type": "keyword" + "index": "not_analyzed", + "type": "string" }, "match_body": { "type": "object", @@ -13,7 +14,8 @@ "format": "dateOptionalTime" }, "aggregate_id": { - "type": "keyword" + "index": "not_analyzed", + "type": "string" } } } diff --git a/elastalert/es_mappings/5/silence.json b/elastalert/es_mappings/5/silence.json index 662008bc5..9d52c6883 100644 --- a/elastalert/es_mappings/5/silence.json +++ b/elastalert/es_mappings/5/silence.json @@ -2,7 +2,8 @@ "silence": { "properties": { "rule_name": { - "type": "keyword" + "index": "not_analyzed", + "type": "string" }, "until": { "type": "date", diff --git a/elastalert/es_mappings/6/elastalert.json b/elastalert/es_mappings/6/elastalert.json index d12861184..6cb997f32 100644 --- a/elastalert/es_mappings/6/elastalert.json +++ b/elastalert/es_mappings/6/elastalert.json @@ -1,8 +1,7 @@ { "properties": { "rule_name": { - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "@timestamp": { "type": "date", @@ -21,8 +20,7 @@ "enabled": "false" }, "aggregate_id": { - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } \ No newline at end of file diff --git a/elastalert/es_mappings/6/elastalert_status.json b/elastalert/es_mappings/6/elastalert_status.json index 2b4f1781c..72839761b 100644 --- a/elastalert/es_mappings/6/elastalert_status.json +++ b/elastalert/es_mappings/6/elastalert_status.json @@ -1,8 +1,7 @@ { "properties": { "rule_name": { - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "@timestamp": { "type": "date", diff --git a/elastalert/es_mappings/6/past_elastalert.json b/elastalert/es_mappings/6/past_elastalert.json index 0d1675f43..fa84c07bf 100644 --- a/elastalert/es_mappings/6/past_elastalert.json +++ b/elastalert/es_mappings/6/past_elastalert.json @@ -1,8 +1,7 @@ { "properties": { "rule_name": { - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "match_body": { "type": "object", @@ -13,8 +12,7 @@ "format": "dateOptionalTime" }, "aggregate_id": { - "index": "not_analyzed", - "type": "string" + "type": "keyword" } } } \ No newline at end of file diff --git a/elastalert/es_mappings/6/silence.json b/elastalert/es_mappings/6/silence.json index c917347cb..40bad056c 100644 --- a/elastalert/es_mappings/6/silence.json +++ b/elastalert/es_mappings/6/silence.json @@ -1,8 +1,7 @@ { "properties": { "rule_name": { - "index": "not_analyzed", - "type": "string" + "type": "keyword" }, "until": { "type": "date", diff --git a/elastalert/rule_from_kibana.py b/elastalert/rule_from_kibana.py index a16d150da..33f3da3a0 100644 --- a/elastalert/rule_from_kibana.py +++ b/elastalert/rule_from_kibana.py @@ -24,7 +24,7 @@ def main(): query = {'query': {'term': {'_id': db_name}}} if is_atleastsixsix(es_version): - # TODO add support for kibana 5 + # TODO check support for kibana 7 # TODO use doc_type='_doc' instead # TODO use _source_includes=[...] instead when elasticsearch client supports this res = es.search(index='kibana-int', doc_type='dashboard', body=query, params={'_source_includes': 'dashboard'}) diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..cb9092f0f --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + elasticsearch: mark a test as using elasticsearch. \ No newline at end of file diff --git a/tests/conftest.py b/tests/conftest.py index d585b1c19..2f9f245f5 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,11 +11,26 @@ from elastalert.util import dt_to_ts from elastalert.util import ts_to_dt - mock_info = {'status': 200, 'name': 'foo', 'version': {'number': '2.0'}} mock_sixsix_info = {'status': 200, 'name': 'foo', 'version': {'number': '6.6.0'}} +def pytest_addoption(parser): + parser.addoption( + "--runelasticsearch", action="store_true", default=False, help="run elasticsearch tests" + ) + + +def pytest_collection_modifyitems(config, items): + if config.getoption("--runelasticsearch"): + # --runelasticsearch given in cli: do not skip elasticsearch tests + return + skip_elasticsearch = pytest.mark.skip(reason="need --runelasticsearch option to run") + for item in items: + if "elasticsearch" in item.keywords: + item.add_marker(skip_elasticsearch) + + @pytest.fixture(scope='function', autouse=True) def reset_loggers(): """Prevent logging handlers from capturing temporary file handles. diff --git a/tests/create_index_test.py b/tests/create_index_test.py index ba306aee5..4b0cc1287 100644 --- a/tests/create_index_test.py +++ b/tests/create_index_test.py @@ -2,6 +2,7 @@ import json import pytest +from elasticsearch import Elasticsearch, RequestsHttpConnection import elastalert.create_index @@ -51,3 +52,29 @@ def test_read_es_6_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings(6) assert len(mappings) == len(es_mappings) print(json.dumps(mappings, indent=2)) + + +@pytest.mark.elasticsearch +def test_create_indices(): + es = Elasticsearch(host='127.0.0.1', port=9200, connection_class=RequestsHttpConnection, timeout=10) + print(json.dumps(es.info()['version']['number'], indent=2)) + index = 'create_index' + elastalert.create_index.main(es_client=es, ea_index=index) + indices_mappings = es.indices.get_mapping(index + '*') + print(json.dumps(indices_mappings, indent=2)) + if es_major_version(es) > 5: + assert index in indices_mappings + assert index + '_error' in indices_mappings + assert index + '_status' in indices_mappings + assert index + '_silence' in indices_mappings + assert index + '_past' in indices_mappings + else: + assert 'elastalert' in indices_mappings[index]['mappings'] + assert 'elastalert_error' in indices_mappings[index]['mappings'] + assert 'elastalert_status' in indices_mappings[index]['mappings'] + assert 'silence' in indices_mappings[index]['mappings'] + assert 'past_elastalert' in indices_mappings[index]['mappings'] + + +def es_major_version(es): + return int(es.info()['version']['number'].split(".")[0]) From a6ec40517a36d706647698f9c29ea690e1dafb14 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Mon, 1 Apr 2019 18:48:35 +0200 Subject: [PATCH 166/264] Travis: DRY avoid repating install and script in all elasticsearch stage jobs --- .editorconfig | 17 +++++++++++++++++ .travis.yml | 35 ++++++++++++++++++----------------- 2 files changed, 35 insertions(+), 17 deletions(-) create mode 100644 .editorconfig diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..d2c3bb958 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 + +[*.py] +indent_style = space +indent_size = 4 + +[Makefile] +indent_style = tab + +[{*.json,*.yml}] +indent_style = space +indent_size = 2 diff --git a/.travis.yml b/.travis.yml index 1f5632943..bf8f0437e 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,26 +6,27 @@ env: - TOXENV=py27 install: - pip install tox -script: make test +- > + if [[ -n "${ES_VERSION}" ]] ; then + wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz + mkdir elasticsearch-${ES_VERSION} && tar -xzf elasticsearch-${ES_VERSION}.tar.gz -C elasticsearch-${ES_VERSION} --strip-components=1 + ./elasticsearch-${ES_VERSION}/bin/elasticsearch & + fi +script: +- > + if [[ -n "${ES_VERSION}" ]] ; then + wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200 + make test-elasticsearch + else + make test + fi jobs: include: - - stage: "Elasticsearch" - install: - - pip install tox - - wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz - - mkdir elasticsearch-${ES_VERSION} && tar -xzf elasticsearch-${ES_VERSION}.tar.gz -C elasticsearch-${ES_VERSION} --strip-components=1 - - ./elasticsearch-${ES_VERSION}/bin/elasticsearch & - script: - - wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200 - - make test-elasticsearch - name: "Elasticsearch 7" + - stage: 'Elasticsearch test' env: TOXENV=py27 ES_VERSION=7.0.0-rc1-linux-x86_64 - - name: "Elasticsearch 6.6" - env: TOXENV=py27 ES_VERSION=6.6.2 - - name: "Elasticsearch 6.3" - env: TOXENV=py27 ES_VERSION=6.3.2 - - name: "Elasticsearch 5.6" - env: TOXENV=py27 ES_VERSION=5.6.16 + - env: TOXENV=py27 ES_VERSION=6.6.2 + - env: TOXENV=py27 ES_VERSION=6.3.2 + - env: TOXENV=py27 ES_VERSION=5.6.16 deploy: provider: pypi From a0e68fe54847411dcc641a26b4c842d493b299a7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Thu, 4 Apr 2019 15:37:19 +0200 Subject: [PATCH 167/264] Extend ElasticSearch and move elasticsearch version checks into extended class elasticsearch_test - test api calls against elasticsearch --- .editorconfig | 2 +- elastalert/elastalert.py | 70 +++++----------- elastalert/elasticsearchclient.py | 44 ++++++++++ elastalert/util.py | 18 +--- tests/base_test.py | 31 ++++--- tests/conftest.py | 37 +++++++- tests/create_index_test.py | 27 ------ tests/elasticsearch_test.py | 135 ++++++++++++++++++++++++++++++ 8 files changed, 254 insertions(+), 110 deletions(-) create mode 100644 elastalert/elasticsearchclient.py create mode 100644 tests/elasticsearch_test.py diff --git a/.editorconfig b/.editorconfig index d2c3bb958..f8eafe691 100644 --- a/.editorconfig +++ b/.editorconfig @@ -12,6 +12,6 @@ indent_size = 4 [Makefile] indent_style = tab -[{*.json,*.yml}] +[{*.json,*.yml,*.yaml}] indent_style = space indent_size = 2 diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 0d8df5f30..a0c51780d 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -123,7 +123,6 @@ def __init__(self, args): self.num_hits = 0 self.num_dupes = 0 self.current_es = None - self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} self.rule_hashes = get_rule_hashes(self.conf, self.args.rule) @@ -134,7 +133,6 @@ def __init__(self, args): self.add_metadata_alert = self.conf.get('add_metadata_alert', False) self.writeback_es = elasticsearch_client(self.conf) - self._es_version = None remove = [] for rule in self.rules: @@ -145,28 +143,6 @@ def __init__(self, args): if self.args.silence: self.silence() - def get_version(self): - info = self.writeback_es.info() - return info['version']['number'] - - @property - def es_version(self): - if self._es_version is None: - self._es_version = self.get_version() - return self._es_version - - def is_atleastfive(self): - return int(self.es_version.split(".")[0]) >= 5 - - def is_atleastsix(self): - return int(self.es_version.split(".")[0]) >= 6 - - def is_atleastsixsix(self): - return float('.'.join(self.es_version.split(".")[:2])) >= 6.6 - - def is_atleastseven(self): - return int(self.es_version.split(".")[0]) >= 7 - @staticmethod def get_index(rule, starttime=None, endtime=None): """ Gets the index for a rule. If strftime is set and starttime and endtime @@ -289,7 +265,7 @@ def get_index_start(self, index, timestamp_field='@timestamp'): """ query = {'sort': {timestamp_field: {'order': 'asc'}}} try: - if self.is_atleastsixsix(): + if self.current_es.is_atleastsixsix(): # TODO use _source_includes=[...] instead when elasticsearch client supports this res = self.current_es.search(index=index, size=1, body=query, params={'_source_includes': timestamp_field}, ignore_unavailable=True) @@ -366,7 +342,7 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): to_ts_func=rule['dt_to_ts'], five=rule['five'], ) - if self.is_atleastsixsix(): + if self.current_es.is_atleastsixsix(): # TODO fix when elasticsearch client supports param _source_includes # Since _source_includes is not supported we must use params instead. # the value object in _source_includes is not automagically parsed into a legal @@ -395,7 +371,7 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): **extra_args ) - if self.is_atleastseven(): + if self.current_es.is_atleastseven(): self.total_hits = int(res['hits']['total']['value']) else: self.total_hits = int(res['hits']['total']) @@ -585,7 +561,7 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ else: payload = res['aggregations'] - if self.is_atleastseven(): + if self.current_es.is_atleastseven(): self.num_hits += res['hits']['total']['value'] else: self.num_hits += res['hits']['total'] @@ -678,14 +654,14 @@ def get_starttime(self, rule): """ sort = {'sort': {'@timestamp': {'order': 'desc'}}} query = {'filter': {'term': {'rule_name': '%s' % (rule['name'])}}} - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): query = {'query': {'bool': query}} query.update(sort) try: - if self.is_atleastsix(): + if self.writeback_es.is_atleastsix(): index = self.get_six_index('elastalert_status') - if self.is_atleastsixsix(): + if self.writeback_es.is_atleastsixsix(): # TODO use _source_includes=[...] instead when elasticsearch client supports this res = self.writeback_es.search(index=index, doc_type='_doc', size=1, body=query, params={'_source_includes': 'endtime,rule_name'}) @@ -843,7 +819,7 @@ def enhance_filter(self, rule): else: query = " OR ".join(additional_terms) query_str_filter = {'query_string': {'query': query}} - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): filters.append(query_str_filter) else: filters.append({'query': query_str_filter}) @@ -859,7 +835,6 @@ def run_rule(self, rule, endtime, starttime=None): """ run_start = time.time() self.current_es = elasticsearch_client(rule) - self.current_es_addr = (rule['es_host'], rule['es_port']) # If there are pending aggregate matches, try processing them for x in range(len(rule['agg_matches'])): @@ -984,7 +959,7 @@ def init_rule(self, new_rule, new=True): if 'top_count_keys' in new_rule and new_rule.get('raw_count_keys', True): if self.string_multi_field_name: string_multi_field_name = self.string_multi_field_name - elif self.is_atleastfive(): + elif self.writeback_es.is_atleastfive(): string_multi_field_name = '.keyword' else: string_multi_field_name = '.raw' @@ -1032,7 +1007,7 @@ def init_rule(self, new_rule, new=True): def modify_rule_for_ES5(new_rule): # Get ES version per rule rule_es = elasticsearch_client(new_rule) - if int(rule_es.info()['version']['number'].split(".")[0]) >= 5: + if rule_es.is_atleastfive(): new_rule['five'] = True else: new_rule['five'] = False @@ -1335,7 +1310,7 @@ def get_dashboard(self, rule, db_name): raise EAException("use_kibana_dashboard undefined") query = {'query': {'term': {'_id': db_name}}} try: - if self.is_atleastsixsix(): + if es.is_atleastsixsix(): # TODO use doc_type = _doc # TODO use _source_includes=[...] instead when elasticsearch client supports for this res = es.search(index='kibana-int', doc_type='dashboard', body=query, @@ -1533,7 +1508,7 @@ def writeback(self, doc_type, body): writeback_body['@timestamp'] = dt_to_ts(ts_now()) try: - if self.is_atleastsix(): + if self.writeback_es.is_atleastsix(): writeback_index = self.get_six_index(doc_type) res = self.writeback_es.index(index=writeback_index, doc_type='_doc', body=body) else: @@ -1554,13 +1529,13 @@ def find_recent_pending_alerts(self, time_limit): time_filter = {'range': {'alert_time': {'from': dt_to_ts(ts_now() - time_limit), 'to': dt_to_ts(ts_now())}}} sort = {'sort': {'alert_time': {'order': 'asc'}}} - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): query = {'query': {'bool': {'must': inner_query, 'filter': time_filter}}} else: query = {'query': inner_query, 'filter': time_filter} query.update(sort) try: - if self.is_atleastsix(): + if self.writeback_es.is_atleastsix(): res = self.writeback_es.search(index=self.writeback_index, doc_type='_doc', body=query, size=1000) else: res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert', body=query, size=1000) @@ -1593,7 +1568,6 @@ def send_pending_alerts(self): # Set current_es for top_count_keys query self.current_es = elasticsearch_client(rule) - self.current_es_addr = (rule['es_host'], rule['es_port']) # Send the alert unless it's a future alert if ts_now() > ts_to_dt(alert_time): @@ -1616,7 +1590,7 @@ def send_pending_alerts(self): # Delete it from the index try: - if self.is_atleastsix(): + if self.writeback_es.is_atleastsix(): self.writeback_es.delete(index=self.writeback_index, doc_type='_doc', id=_id) else: self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=_id) @@ -1649,7 +1623,7 @@ def get_aggregated_matches(self, _id): query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}} matches = [] try: - if self.is_atleastsix(): + if self.writeback_es.is_atleastsix(): res = self.writeback_es.search(index=self.writeback_index, doc_type='_doc', body=query, size=self.max_aggregation) else: @@ -1657,7 +1631,7 @@ def get_aggregated_matches(self, _id): size=self.max_aggregation) for match in res['hits']['hits']: matches.append(match['_source']) - if self.is_atleastsix(): + if self.writeback_es.is_atleastsix(): self.writeback_es.delete(index=self.writeback_index, doc_type='_doc', id=match['_id']) else: self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=match['_id']) @@ -1672,11 +1646,11 @@ def find_pending_aggregate_alert(self, rule, aggregation_key_value=None): 'must_not': [{'exists': {'field': 'aggregate_id'}}]}}} if aggregation_key_value: query['filter']['bool']['must'].append({'term': {'aggregation_key': aggregation_key_value}}) - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): query = {'query': {'bool': query}} query['sort'] = {'alert_time': {'order': 'desc'}} try: - if self.is_atleastsix(): + if self.writeback_es.is_atleastsix(): res = self.writeback_es.search(index=self.writeback_index, doc_type='_doc', body=query, size=1) else: res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert', body=query, size=1) @@ -1811,16 +1785,16 @@ def is_silenced(self, rule_name): return False query = {'term': {'rule_name': rule_name}} sort = {'sort': {'until': {'order': 'desc'}}} - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): query = {'query': query} else: query = {'filter': query} query.update(sort) try: - if self.is_atleastsix(): + if self.writeback_es.is_atleastsix(): index = self.get_six_index('silence') - if self.is_atleastsixsix(): + if self.writeback_es.is_atleastsixsix(): # TODO use _source_includes=[...] instead when elasticsearch client supports this res = self.writeback_es.search(index=index, doc_type='_doc', size=1, body=query, params={'_source_includes': 'until,exponent'}) diff --git a/elastalert/elasticsearchclient.py b/elastalert/elasticsearchclient.py new file mode 100644 index 000000000..f8f8061bf --- /dev/null +++ b/elastalert/elasticsearchclient.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from elasticsearch import Elasticsearch, RequestsHttpConnection + + +class ElasticSearchClient(Elasticsearch): + + def __init__(self, conf): + super(ElasticSearchClient, self).__init__(host=conf['es_host'], + port=conf['es_port'], + url_prefix=conf['es_url_prefix'], + use_ssl=conf['use_ssl'], + verify_certs=conf['verify_certs'], + ca_certs=conf['ca_certs'], + connection_class=RequestsHttpConnection, + http_auth=conf['http_auth'], + timeout=conf['es_conn_timeout'], + send_get_body_as=conf['send_get_body_as'], + client_cert=conf['client_cert'], + client_key=conf['client_key']) + self._conf = conf + self._es_version = None + + @property + def conf(self): + return self._conf + + @property + def es_version(self): + if self._es_version is None: + self._es_version = self.info()['version']['number'] + return self._es_version + + def is_atleastfive(self): + return int(self.es_version.split(".")[0]) >= 5 + + def is_atleastsix(self): + return int(self.es_version.split(".")[0]) >= 6 + + def is_atleastsixsix(self): + major, minor = map(int, self.es_version.split(".")[:2]) + return major > 6 or (major == 6 and minor >= 6) + + def is_atleastseven(self): + return int(self.es_version.split(".")[0]) >= 7 diff --git a/elastalert/util.py b/elastalert/util.py index 33f0b4e71..47238cf6c 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -7,8 +7,7 @@ import dateutil.parser import dateutil.tz from auth import Auth -from elasticsearch import RequestsHttpConnection -from elasticsearch.client import Elasticsearch +import elasticsearchclient from six import string_types logging.basicConfig() @@ -281,7 +280,7 @@ def replace_dots_in_field_names(document): def elasticsearch_client(conf): - """ returns an Elasticsearch instance configured using an es_conn_config """ + """ returns an ElasticsearchClient instance configured using an es_conn_config """ es_conn_conf = build_es_conn_config(conf) auth = Auth() es_conn_conf['http_auth'] = auth(host=es_conn_conf['es_host'], @@ -290,18 +289,7 @@ def elasticsearch_client(conf): aws_region=es_conn_conf['aws_region'], profile_name=es_conn_conf['profile']) - return Elasticsearch(host=es_conn_conf['es_host'], - port=es_conn_conf['es_port'], - url_prefix=es_conn_conf['es_url_prefix'], - use_ssl=es_conn_conf['use_ssl'], - verify_certs=es_conn_conf['verify_certs'], - ca_certs=es_conn_conf['ca_certs'], - connection_class=RequestsHttpConnection, - http_auth=es_conn_conf['http_auth'], - timeout=es_conn_conf['es_conn_timeout'], - send_get_body_as=es_conn_conf['send_get_body_as'], - client_cert=es_conn_conf['client_cert'], - client_key=es_conn_conf['client_key']) + return elasticsearchclient.ElasticSearchClient(es_conn_conf) def build_es_conn_config(conf): diff --git a/tests/base_test.py b/tests/base_test.py index efd986d10..559ed1f85 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1324,14 +1324,13 @@ def test_query_with_whitelist_filter_es(ea): in new_rule['filter'][-1]['query']['query_string']['query'] -def test_query_with_whitelist_filter_es_five(ea): - ea.es_version = '6.2' - ea.rules[0]['_source_enabled'] = False - ea.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}] - ea.rules[0]['compare_key'] = "username" - ea.rules[0]['whitelist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] - new_rule = copy.copy(ea.rules[0]) - ea.init_rule(new_rule, True) +def test_query_with_whitelist_filter_es_five(ea_sixsix): + ea_sixsix.rules[0]['_source_enabled'] = False + ea_sixsix.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}] + ea_sixsix.rules[0]['compare_key'] = "username" + ea_sixsix.rules[0]['whitelist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] + new_rule = copy.copy(ea_sixsix.rules[0]) + ea_sixsix.init_rule(new_rule, True) assert 'NOT username:"xudan1" AND NOT username:"xudan12" AND NOT username:"aa1"' in \ new_rule['filter'][-1]['query_string']['query'] @@ -1347,13 +1346,13 @@ def test_query_with_blacklist_filter_es(ea): new_rule['filter'][-1]['query']['query_string']['query'] -def test_query_with_blacklist_filter_es_five(ea): - ea.es_version = '6.2' - ea.rules[0]['_source_enabled'] = False - ea.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}] - ea.rules[0]['compare_key'] = "username" - ea.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] - new_rule = copy.copy(ea.rules[0]) - ea.init_rule(new_rule, True) +def test_query_with_blacklist_filter_es_five(ea_sixsix): + ea_sixsix.rules[0]['_source_enabled'] = False + ea_sixsix.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}] + ea_sixsix.rules[0]['compare_key'] = "username" + ea_sixsix.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] + ea_sixsix.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] + new_rule = copy.copy(ea_sixsix.rules[0]) + ea_sixsix.init_rule(new_rule, True) assert 'username:"xudan1" OR username:"xudan12" OR username:"aa1"' in new_rule['filter'][-1]['query_string'][ 'query'] diff --git a/tests/conftest.py b/tests/conftest.py index 2f9f245f5..e58909d33 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,9 +11,6 @@ from elastalert.util import dt_to_ts from elastalert.util import ts_to_dt -mock_info = {'status': 200, 'name': 'foo', 'version': {'number': '2.0'}} -mock_sixsix_info = {'status': 200, 'name': 'foo', 'version': {'number': '6.6.0'}} - def pytest_addoption(parser): parser.addoption( @@ -55,6 +52,7 @@ def __init__(self): class mock_es_client(object): def __init__(self, host='es', port=14900): + mock_info = {'status': 200, 'name': 'foo', 'version': {'number': '2.0'}} self.host = host self.port = port self.return_hits = [] @@ -66,9 +64,26 @@ def __init__(self, host='es', port=14900): self.ping = mock.Mock(return_value=True) self.indices = mock_es_indices_client() + @property + def es_version(self): + return self.info()['version']['number'] + + def is_atleastfive(self): + return False + + def is_atleastsix(self): + return False + + def is_atleastsixsix(self): + return False + + def is_atleastseven(self): + return False + class mock_es_sixsix_client(object): def __init__(self, host='es', port=14900): + mock_sixsix_info = {'status': 200, 'name': 'foo', 'version': {'number': '6.6.0'}} self.host = host self.port = port self.return_hits = [] @@ -80,6 +95,22 @@ def __init__(self, host='es', port=14900): self.ping = mock.Mock(return_value=True) self.indices = mock_es_indices_client() + @property + def es_version(self): + return self.info()['version']['number'] + + def is_atleastfive(self): + return True + + def is_atleastsix(self): + return True + + def is_atleastsixsix(self): + return True + + def is_atleastseven(self): + return False + class mock_ruletype(object): def __init__(self): diff --git a/tests/create_index_test.py b/tests/create_index_test.py index 4b0cc1287..ba306aee5 100644 --- a/tests/create_index_test.py +++ b/tests/create_index_test.py @@ -2,7 +2,6 @@ import json import pytest -from elasticsearch import Elasticsearch, RequestsHttpConnection import elastalert.create_index @@ -52,29 +51,3 @@ def test_read_es_6_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings(6) assert len(mappings) == len(es_mappings) print(json.dumps(mappings, indent=2)) - - -@pytest.mark.elasticsearch -def test_create_indices(): - es = Elasticsearch(host='127.0.0.1', port=9200, connection_class=RequestsHttpConnection, timeout=10) - print(json.dumps(es.info()['version']['number'], indent=2)) - index = 'create_index' - elastalert.create_index.main(es_client=es, ea_index=index) - indices_mappings = es.indices.get_mapping(index + '*') - print(json.dumps(indices_mappings, indent=2)) - if es_major_version(es) > 5: - assert index in indices_mappings - assert index + '_error' in indices_mappings - assert index + '_status' in indices_mappings - assert index + '_silence' in indices_mappings - assert index + '_past' in indices_mappings - else: - assert 'elastalert' in indices_mappings[index]['mappings'] - assert 'elastalert_error' in indices_mappings[index]['mappings'] - assert 'elastalert_status' in indices_mappings[index]['mappings'] - assert 'silence' in indices_mappings[index]['mappings'] - assert 'past_elastalert' in indices_mappings[index]['mappings'] - - -def es_major_version(es): - return int(es.info()['version']['number'].split(".")[0]) diff --git a/tests/elasticsearch_test.py b/tests/elasticsearch_test.py new file mode 100644 index 000000000..3dcedb83e --- /dev/null +++ b/tests/elasticsearch_test.py @@ -0,0 +1,135 @@ +# -*- coding: utf-8 -*- +import datetime +import json +import time + +import dateutil +import mock +import pytest + +import elastalert.elasticsearchclient +import elastalert.create_index +import elastalert.elastalert +from elastalert.util import ts_to_dt, dt_to_ts, build_es_conn_config +from tests.conftest import mock_ruletype, mock_alert, mock_es_client + +test_index = 'test_index' + +es_host = '127.0.0.1' +es_port = 9200 +es_timeout = 10 + + +@pytest.fixture +def es_client(): + es_conn_config = build_es_conn_config({'es_host': es_host, 'es_port': es_port, 'es_conn_timeout': es_timeout}) + return elastalert.elasticsearchclient.ElasticSearchClient(es_conn_config) + + +@pytest.fixture +def ea(): + rules = [{'es_host': '', + 'es_port': 14900, + 'name': 'anytest', + 'index': 'idx', + 'filter': [], + 'include': ['@timestamp'], + 'aggregation': datetime.timedelta(0), + 'realert': datetime.timedelta(0), + 'processed_hits': {}, + 'timestamp_field': '@timestamp', + 'match_enhancements': [], + 'rule_file': 'blah.yaml', + 'max_query_size': 10000, + 'ts_to_dt': ts_to_dt, + 'dt_to_ts': dt_to_ts, + '_source_enabled': True}] + conf = {'rules_folder': 'rules', + 'run_every': datetime.timedelta(minutes=10), + 'buffer_time': datetime.timedelta(minutes=5), + 'alert_time_limit': datetime.timedelta(hours=24), + 'es_host': es_host, + 'es_port': es_port, + 'es_conn_timeout': es_timeout, + 'writeback_index': test_index, + 'rules': rules, + 'max_query_size': 10000, + 'old_query_limit': datetime.timedelta(weeks=1), + 'disable_rules_on_error': False, + 'scroll_keepalive': '30s'} + elastalert.elastalert.elasticsearch_client = mock_es_client + with mock.patch('elastalert.elastalert.get_rule_hashes'): + with mock.patch('elastalert.elastalert.load_rules') as load_conf: + load_conf.return_value = conf + ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) + ea.rules[0]['type'] = mock_ruletype() + ea.rules[0]['alert'] = [mock_alert()] + ea.writeback_es = es_client() + ea.current_es = mock_es_client('', '') + return ea + + +@pytest.mark.elasticsearch +class TestElasticsearch: + def test_create_indices(self, es_client): + elastalert.create_index.main(es_client=es_client, ea_index=test_index) + indices_mappings = es_client.indices.get_mapping(test_index + '*') + print('-' * 50) + print(json.dumps(indices_mappings, indent=2)) + print('-' * 50) + if es_client.is_atleastsix(): + assert test_index in indices_mappings + assert test_index + '_error' in indices_mappings + assert test_index + '_status' in indices_mappings + assert test_index + '_silence' in indices_mappings + assert test_index + '_past' in indices_mappings + else: + assert 'elastalert' in indices_mappings[test_index]['mappings'] + assert 'elastalert_error' in indices_mappings[test_index]['mappings'] + assert 'elastalert_status' in indices_mappings[test_index]['mappings'] + assert 'silence' in indices_mappings[test_index]['mappings'] + assert 'past_elastalert' in indices_mappings[test_index]['mappings'] + + def test_aggregated_alert(self, ea): + match_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( + days=1) + ea.rules[0]['aggregate_by_match_time'] = True + match = {'@timestamp': match_timestamp, + 'num_hits': 0, + 'num_matches': 3 + } + res = ea.add_aggregated_alert(match, ea.rules[0]) + if ea.writeback_es.is_atleastsix(): + assert res['result'] == 'created' + else: + assert res['created'] is True + # Make sure added data is available for querying + time.sleep(2) + # Now lets find the pending aggregated alert + assert ea.find_pending_aggregate_alert(ea.rules[0]) + + def test_silenced(self, ea): + until_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( + days=1) + res = ea.set_realert(ea.rules[0]['name'], until_timestamp, 0) + if ea.writeback_es.is_atleastsix(): + assert res['result'] == 'created' + else: + assert res['created'] is True + # Make sure added data is available for querying + time.sleep(2) + # Force lookup in elasticsearch + ea.silence_cache = {} + # Now lets check if our rule is reported as silenced + assert ea.is_silenced(ea.rules[0]['name']) + + def test_get_hits(self, ea, es_client): + start = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + end = start + datetime.timedelta(days=1) + ea.current_es = es_client + if ea.current_es.is_atleastfive(): + ea.rules[0]['five'] = True + else: + ea.rules[0]['five'] = False + hits = ea.get_hits(ea.rules[0], start, end, test_index) + assert isinstance(hits, list) From fa0804ffc34833e1cc4b82ada754be56b07eaf48 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Fri, 5 Apr 2019 09:12:01 +0200 Subject: [PATCH 168/264] Moved ElasticSearchClient into package, added docstrings, skip ordinary unit tests when pytest is run with --runelasticsearch --- elastalert/__init__.py | 67 +++++++++++++++++++++++++++++++ elastalert/elastalert.py | 2 +- elastalert/elasticsearchclient.py | 44 -------------------- elastalert/util.py | 6 +-- tests/conftest.py | 23 ++++++----- tests/elasticsearch_test.py | 10 +++-- 6 files changed, 91 insertions(+), 61 deletions(-) delete mode 100644 elastalert/elasticsearchclient.py diff --git a/elastalert/__init__.py b/elastalert/__init__.py index e69de29bb..2aabf47e2 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +import copy +from elasticsearch import Elasticsearch, RequestsHttpConnection + + +class ElasticSearchClient(Elasticsearch): + """ Extension of low level :class:`Elasticsearch` client with additional version resolving features """ + + def __init__(self, conf): + """ + :arg conf: es_conn_config dictionary. Ref. :func:`~util.build_es_conn_config` + """ + super(ElasticSearchClient, self).__init__(host=conf['es_host'], + port=conf['es_port'], + url_prefix=conf['es_url_prefix'], + use_ssl=conf['use_ssl'], + verify_certs=conf['verify_certs'], + ca_certs=conf['ca_certs'], + connection_class=RequestsHttpConnection, + http_auth=conf['http_auth'], + timeout=conf['es_conn_timeout'], + send_get_body_as=conf['send_get_body_as'], + client_cert=conf['client_cert'], + client_key=conf['client_key']) + self._conf = copy.copy(conf) + self._es_version = None + + @property + def conf(self): + """ + Returns the provided es_conn_config used when initializing the class instance. + """ + return self._conf + + @property + def es_version(self): + """ + Returns the reported version from the Elasticsearch server. + """ + if self._es_version is None: + self._es_version = self.info()['version']['number'] + return self._es_version + + def is_atleastfive(self): + """ + Returns True when the Elasticsearch server version >= 5 + """ + return int(self.es_version.split(".")[0]) >= 5 + + def is_atleastsix(self): + """ + Returns True when the Elasticsearch server version >= 6 + """ + return int(self.es_version.split(".")[0]) >= 6 + + def is_atleastsixsix(self): + """ + Returns True when the Elasticsearch server version >= 6.6 + """ + major, minor = map(int, self.es_version.split(".")[:2]) + return major > 6 or (major == 6 and minor >= 6) + + def is_atleastseven(self): + """ + Returns True when the Elasticsearch server version >= 7 + """ + return int(self.es_version.split(".")[0]) >= 7 diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index a0c51780d..ec799e258 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -50,7 +50,7 @@ from util import unix_to_dt -class ElastAlerter(): +class ElastAlerter(object): """ The main ElastAlert runner. This class holds all state about active rules, controls when queries are run, and passes information between rules and alerts. diff --git a/elastalert/elasticsearchclient.py b/elastalert/elasticsearchclient.py deleted file mode 100644 index f8f8061bf..000000000 --- a/elastalert/elasticsearchclient.py +++ /dev/null @@ -1,44 +0,0 @@ -# -*- coding: utf-8 -*- -from elasticsearch import Elasticsearch, RequestsHttpConnection - - -class ElasticSearchClient(Elasticsearch): - - def __init__(self, conf): - super(ElasticSearchClient, self).__init__(host=conf['es_host'], - port=conf['es_port'], - url_prefix=conf['es_url_prefix'], - use_ssl=conf['use_ssl'], - verify_certs=conf['verify_certs'], - ca_certs=conf['ca_certs'], - connection_class=RequestsHttpConnection, - http_auth=conf['http_auth'], - timeout=conf['es_conn_timeout'], - send_get_body_as=conf['send_get_body_as'], - client_cert=conf['client_cert'], - client_key=conf['client_key']) - self._conf = conf - self._es_version = None - - @property - def conf(self): - return self._conf - - @property - def es_version(self): - if self._es_version is None: - self._es_version = self.info()['version']['number'] - return self._es_version - - def is_atleastfive(self): - return int(self.es_version.split(".")[0]) >= 5 - - def is_atleastsix(self): - return int(self.es_version.split(".")[0]) >= 6 - - def is_atleastsixsix(self): - major, minor = map(int, self.es_version.split(".")[:2]) - return major > 6 or (major == 6 and minor >= 6) - - def is_atleastseven(self): - return int(self.es_version.split(".")[0]) >= 7 diff --git a/elastalert/util.py b/elastalert/util.py index 47238cf6c..8da4c0e51 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -7,7 +7,7 @@ import dateutil.parser import dateutil.tz from auth import Auth -import elasticsearchclient +from . import ElasticSearchClient from six import string_types logging.basicConfig() @@ -280,7 +280,7 @@ def replace_dots_in_field_names(document): def elasticsearch_client(conf): - """ returns an ElasticsearchClient instance configured using an es_conn_config """ + """ returns an :class:`ElasticSearchClient` instance configured using an es_conn_config """ es_conn_conf = build_es_conn_config(conf) auth = Auth() es_conn_conf['http_auth'] = auth(host=es_conn_conf['es_host'], @@ -289,7 +289,7 @@ def elasticsearch_client(conf): aws_region=es_conn_conf['aws_region'], profile_name=es_conn_conf['profile']) - return elasticsearchclient.ElasticSearchClient(es_conn_conf) + return ElasticSearchClient(es_conn_conf) def build_es_conn_config(conf): diff --git a/tests/conftest.py b/tests/conftest.py index e58909d33..2f91aa5a7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -20,12 +20,17 @@ def pytest_addoption(parser): def pytest_collection_modifyitems(config, items): if config.getoption("--runelasticsearch"): - # --runelasticsearch given in cli: do not skip elasticsearch tests - return - skip_elasticsearch = pytest.mark.skip(reason="need --runelasticsearch option to run") - for item in items: - if "elasticsearch" in item.keywords: - item.add_marker(skip_elasticsearch) + # --runelasticsearch given in cli: run elasticsearch tests, skip ordinary unit tests + skip_unit_tests = pytest.mark.skip(reason="not running when --runelasticsearch option is used to run") + for item in items: + if "elasticsearch" not in item.keywords: + item.add_marker(skip_unit_tests) + else: + # skip elasticsearch tests + skip_elasticsearch = pytest.mark.skip(reason="need --runelasticsearch option to run") + for item in items: + if "elasticsearch" in item.keywords: + item.add_marker(skip_elasticsearch) @pytest.fixture(scope='function', autouse=True) @@ -52,7 +57,6 @@ def __init__(self): class mock_es_client(object): def __init__(self, host='es', port=14900): - mock_info = {'status': 200, 'name': 'foo', 'version': {'number': '2.0'}} self.host = host self.port = port self.return_hits = [] @@ -60,7 +64,7 @@ def __init__(self, host='es', port=14900): self.create = mock.Mock() self.index = mock.Mock() self.delete = mock.Mock() - self.info = mock.Mock(return_value=mock_info) + self.info = mock.Mock(return_value={'status': 200, 'name': 'foo', 'version': {'number': '2.0'}}) self.ping = mock.Mock(return_value=True) self.indices = mock_es_indices_client() @@ -83,7 +87,6 @@ def is_atleastseven(self): class mock_es_sixsix_client(object): def __init__(self, host='es', port=14900): - mock_sixsix_info = {'status': 200, 'name': 'foo', 'version': {'number': '6.6.0'}} self.host = host self.port = port self.return_hits = [] @@ -91,7 +94,7 @@ def __init__(self, host='es', port=14900): self.create = mock.Mock() self.index = mock.Mock() self.delete = mock.Mock() - self.info = mock.Mock(return_value=mock_sixsix_info) + self.info = mock.Mock(return_value={'status': 200, 'name': 'foo', 'version': {'number': '6.6.0'}}) self.ping = mock.Mock(return_value=True) self.indices = mock_es_indices_client() diff --git a/tests/elasticsearch_test.py b/tests/elasticsearch_test.py index 3dcedb83e..c293fdf05 100644 --- a/tests/elasticsearch_test.py +++ b/tests/elasticsearch_test.py @@ -7,9 +7,9 @@ import mock import pytest -import elastalert.elasticsearchclient import elastalert.create_index import elastalert.elastalert +from elastalert import ElasticSearchClient from elastalert.util import ts_to_dt, dt_to_ts, build_es_conn_config from tests.conftest import mock_ruletype, mock_alert, mock_es_client @@ -23,7 +23,7 @@ @pytest.fixture def es_client(): es_conn_config = build_es_conn_config({'es_host': es_host, 'es_port': es_port, 'es_conn_timeout': es_timeout}) - return elastalert.elasticsearchclient.ElasticSearchClient(es_conn_config) + return ElasticSearchClient(es_conn_config) @pytest.fixture @@ -70,7 +70,11 @@ def ea(): @pytest.mark.elasticsearch -class TestElasticsearch: +class TestElasticsearch(object): + # TODO perform teardown removing data inserted into Elasticsearch + # Warning!!!: Test class is not erasing its testdata on the Elasticsearch server. + # This is not a problem as long as the data is manually removed or the test environment + # is torn down after the test run(eg. running tests in a test environment such as Travis) def test_create_indices(self, es_client): elastalert.create_index.main(es_client=es_client, ea_index=test_index) indices_mappings = es_client.indices.get_mapping(test_index + '*') From 1fbd5ddc25aff81cdca00799d1132579cc6d5acb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Fri, 5 Apr 2019 12:39:19 +0200 Subject: [PATCH 169/264] Revert travis repo badge back to Yelp/elastalert --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0900957e9..1bfa1f2ea 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ [![Stories in Ready](https://badge.waffle.io/Yelp/elastalert.png?label=ready&title=Ready)](https://waffle.io/Yelp/elastalert) [![Stories in In Progress](https://badge.waffle.io/Yelp/elastalert.png?label=in%20progress&title=In%20Progress)](https://waffle.io/Yelp/elastalert) -[![Build Status](https://travis-ci.com/matsgoran/elastalert.svg?branch=master)](https://travis-ci.com/matsgoran/elastalert) +[![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) ## ElastAlert - [Read the Docs](http://elastalert.readthedocs.org). From 5b9592b2ddac851bcf9d40a2c73cc486d46144f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Fri, 5 Apr 2019 12:45:20 +0200 Subject: [PATCH 170/264] Fixup files with no new line at end of file --- elastalert/es_mappings/5/elastalert.json | 2 +- elastalert/es_mappings/5/elastalert_error.json | 2 +- elastalert/es_mappings/5/elastalert_status.json | 2 +- elastalert/es_mappings/5/past_elastalert.json | 2 +- elastalert/es_mappings/5/silence.json | 2 +- elastalert/es_mappings/6/elastalert.json | 2 +- elastalert/es_mappings/6/elastalert_error.json | 2 +- elastalert/es_mappings/6/elastalert_status.json | 2 +- elastalert/es_mappings/6/past_elastalert.json | 2 +- elastalert/es_mappings/6/silence.json | 2 +- pytest.ini | 2 +- 11 files changed, 11 insertions(+), 11 deletions(-) diff --git a/elastalert/es_mappings/5/elastalert.json b/elastalert/es_mappings/5/elastalert.json index 5fbdebc6c..b522933b3 100644 --- a/elastalert/es_mappings/5/elastalert.json +++ b/elastalert/es_mappings/5/elastalert.json @@ -27,4 +27,4 @@ } } } -} \ No newline at end of file +} diff --git a/elastalert/es_mappings/5/elastalert_error.json b/elastalert/es_mappings/5/elastalert_error.json index 8672dbd94..7f1b3c0a8 100644 --- a/elastalert/es_mappings/5/elastalert_error.json +++ b/elastalert/es_mappings/5/elastalert_error.json @@ -11,4 +11,4 @@ } } } -} \ No newline at end of file +} diff --git a/elastalert/es_mappings/5/elastalert_status.json b/elastalert/es_mappings/5/elastalert_status.json index 597ea35a8..f8cd9643f 100644 --- a/elastalert/es_mappings/5/elastalert_status.json +++ b/elastalert/es_mappings/5/elastalert_status.json @@ -11,4 +11,4 @@ } } } -} \ No newline at end of file +} diff --git a/elastalert/es_mappings/5/past_elastalert.json b/elastalert/es_mappings/5/past_elastalert.json index 9fc943a1b..e10783748 100644 --- a/elastalert/es_mappings/5/past_elastalert.json +++ b/elastalert/es_mappings/5/past_elastalert.json @@ -19,4 +19,4 @@ } } } -} \ No newline at end of file +} diff --git a/elastalert/es_mappings/5/silence.json b/elastalert/es_mappings/5/silence.json index 9d52c6883..b04006da8 100644 --- a/elastalert/es_mappings/5/silence.json +++ b/elastalert/es_mappings/5/silence.json @@ -15,4 +15,4 @@ } } } -} \ No newline at end of file +} diff --git a/elastalert/es_mappings/6/elastalert.json b/elastalert/es_mappings/6/elastalert.json index 6cb997f32..ecc30c8ed 100644 --- a/elastalert/es_mappings/6/elastalert.json +++ b/elastalert/es_mappings/6/elastalert.json @@ -23,4 +23,4 @@ "type": "keyword" } } -} \ No newline at end of file +} diff --git a/elastalert/es_mappings/6/elastalert_error.json b/elastalert/es_mappings/6/elastalert_error.json index c7be1dc13..b4b577c16 100644 --- a/elastalert/es_mappings/6/elastalert_error.json +++ b/elastalert/es_mappings/6/elastalert_error.json @@ -9,4 +9,4 @@ "format": "dateOptionalTime" } } -} \ No newline at end of file +} diff --git a/elastalert/es_mappings/6/elastalert_status.json b/elastalert/es_mappings/6/elastalert_status.json index 72839761b..eea1762af 100644 --- a/elastalert/es_mappings/6/elastalert_status.json +++ b/elastalert/es_mappings/6/elastalert_status.json @@ -8,4 +8,4 @@ "format": "dateOptionalTime" } } -} \ No newline at end of file +} diff --git a/elastalert/es_mappings/6/past_elastalert.json b/elastalert/es_mappings/6/past_elastalert.json index fa84c07bf..0cf2c67db 100644 --- a/elastalert/es_mappings/6/past_elastalert.json +++ b/elastalert/es_mappings/6/past_elastalert.json @@ -15,4 +15,4 @@ "type": "keyword" } } -} \ No newline at end of file +} diff --git a/elastalert/es_mappings/6/silence.json b/elastalert/es_mappings/6/silence.json index 40bad056c..d30c2b066 100644 --- a/elastalert/es_mappings/6/silence.json +++ b/elastalert/es_mappings/6/silence.json @@ -12,4 +12,4 @@ "format": "dateOptionalTime" } } -} \ No newline at end of file +} diff --git a/pytest.ini b/pytest.ini index cb9092f0f..0ad3341d9 100644 --- a/pytest.ini +++ b/pytest.ini @@ -1,3 +1,3 @@ [pytest] markers = - elasticsearch: mark a test as using elasticsearch. \ No newline at end of file + elasticsearch: mark a test as using elasticsearch. From 340930e7e0e7d80611b095964a8d2e4551584002 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Fri, 5 Apr 2019 14:08:50 +0200 Subject: [PATCH 171/264] Enhance es version check in rule_from_kibana.py --- elastalert/rule_from_kibana.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elastalert/rule_from_kibana.py b/elastalert/rule_from_kibana.py index 33f3da3a0..07310c8dc 100644 --- a/elastalert/rule_from_kibana.py +++ b/elastalert/rule_from_kibana.py @@ -48,7 +48,8 @@ def main(): def is_atleastsixsix(es_version): - return float('.'.join(es_version.split('.')[:2])) >= 6.6 + major, minor = map(int, es_version.split(".")[:2]) + return major > 6 or (major == 6 and minor >= 6) if __name__ == '__main__': From 1c0d0eb39db77d0018503a7e1dce2a10ecb53626 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Mon, 8 Apr 2019 12:50:18 +0200 Subject: [PATCH 172/264] Yaml Loader added to avoid deprecation warnings --- elastalert/config.py | 2 +- elastalert/create_index.py | 2 +- elastalert/elastalert.py | 2 +- elastalert/test_rule.py | 4 ++-- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index af72e5728..7c1ab69f8 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -28,7 +28,7 @@ from util import unixms_to_dt # schema for rule yaml -rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) +rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml'))), Loader=yaml.FullLoader) # Required global (config.yaml) and local (rule.yaml) configuration options required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 86baf6ab2..881ee9b3b 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -60,7 +60,7 @@ def main(): if filename: with open(filename) as config_file: - data = yaml.load(config_file) + data = yaml.load(config_file, Loader=yaml.FullLoader) host = args.host if args.host else data.get('es_host') port = args.port if args.port else data.get('es_port') username = args.username if args.username else data.get('es_username') diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index d5bee92ad..b5004c686 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1038,7 +1038,7 @@ def load_rule_changes(self): # Want to send email to address specified in the rule. Try and load the YAML to find it. with open(rule_file) as f: try: - rule_yaml = yaml.load(f) + rule_yaml = yaml.load(f, Loader=yaml.FullLoader) except yaml.scanner.ScannerError: self.send_notification_email(exception=e) continue diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 41b55dd95..a9d889b28 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -336,11 +336,11 @@ def load_conf(self, rules, args): :return: the default rule configuration, a dictionary """ if args.config is not None: with open(args.config) as fh: - conf = yaml.load(fh) + conf = yaml.load(fh, Loader=yaml.FullLoader) else: if os.path.isfile('config.yaml'): with open('config.yaml') as fh: - conf = yaml.load(fh) + conf = yaml.load(fh, Loader=yaml.FullLoader) else: conf = {} From 0f33e0ce5825fa916dd60b0f6aec991173762cc5 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Mon, 8 Apr 2019 13:25:19 +0200 Subject: [PATCH 173/264] Bug fixed --- elastalert/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/config.py b/elastalert/config.py index 7c1ab69f8..b41730108 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -28,7 +28,7 @@ from util import unixms_to_dt # schema for rule yaml -rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml'))), Loader=yaml.FullLoader) +rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader)) # Required global (config.yaml) and local (rule.yaml) configuration options required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) From 33e6ab5204088b0ef2c2c05ce41b825c5844e072 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Fri, 12 Apr 2019 12:08:36 +0200 Subject: [PATCH 174/264] Updated to elasticsearch-py==7 - elasticsearch>=7.0.0 in dependencies - fixed api changes in elastalert - added deprecated_search method to es-wrapper-client, which accepts doc_type parameter, which works as an adapter between elastalert and elasticsearch-py --- .travis.yml | 3 +- elastalert/__init__.py | 162 +++++++++++++++++++++++++++++++++ elastalert/create_index.py | 28 ++++-- elastalert/elastalert.py | 71 ++++++--------- elastalert/rule_from_kibana.py | 20 ++-- requirements.txt | 2 +- setup.py | 2 +- tests/base_test.py | 69 +++++++------- tests/conftest.py | 48 ++++------ 9 files changed, 274 insertions(+), 131 deletions(-) diff --git a/.travis.yml b/.travis.yml index bf8f0437e..c3a8b6a37 100644 --- a/.travis.yml +++ b/.travis.yml @@ -23,9 +23,10 @@ script: jobs: include: - stage: 'Elasticsearch test' - env: TOXENV=py27 ES_VERSION=7.0.0-rc1-linux-x86_64 + env: TOXENV=py27 ES_VERSION=7.0.0-linux-x86_64 - env: TOXENV=py27 ES_VERSION=6.6.2 - env: TOXENV=py27 ES_VERSION=6.3.2 + - env: TOXENV=py27 ES_VERSION=6.0.1 - env: TOXENV=py27 ES_VERSION=5.6.16 deploy: diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 2aabf47e2..7716c253a 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -1,6 +1,8 @@ # -*- coding: utf-8 -*- import copy +import logging from elasticsearch import Elasticsearch, RequestsHttpConnection +from elasticsearch.client import query_params, _make_path class ElasticSearchClient(Elasticsearch): @@ -53,6 +55,13 @@ def is_atleastsix(self): """ return int(self.es_version.split(".")[0]) >= 6 + def is_atleastsixtwo(self): + """ + Returns True when the Elasticsearch server version >= 6.2 + """ + major, minor = map(int, self.es_version.split(".")[:2]) + return major > 6 or (major == 6 and minor >= 2) + def is_atleastsixsix(self): """ Returns True when the Elasticsearch server version >= 6.6 @@ -65,3 +74,156 @@ def is_atleastseven(self): Returns True when the Elasticsearch server version >= 7 """ return int(self.es_version.split(".")[0]) >= 7 + + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "allow_no_indices", + "allow_partial_search_results", + "analyze_wildcard", + "analyzer", + "batched_reduce_size", + "default_operator", + "df", + "docvalue_fields", + "expand_wildcards", + "explain", + "from_", + "ignore_unavailable", + "lenient", + "max_concurrent_shard_requests", + "pre_filter_shard_size", + "preference", + "q", + "rest_total_hits_as_int", + "request_cache", + "routing", + "scroll", + "search_type", + "seq_no_primary_term", + "size", + "sort", + "stats", + "stored_fields", + "suggest_field", + "suggest_mode", + "suggest_size", + "suggest_text", + "terminate_after", + "timeout", + "track_scores", + "track_total_hits", + "typed_keys", + "version", + ) + def deprecated_search(self, index=None, doc_type=None, body=None, params=None): + """ + Execute a search query and get back search hits that match the query. + ``_ + :arg index: A list of index names to search, or a string containing a + comma-separated list of index names to search; use `_all` + or empty string to perform the operation on all indices + :arg doc_type: A comma-separated list of document types to search; leave + empty to perform the operation on all types + :arg body: The search definition using the Query DSL + :arg _source: True or false to return the _source field or not, or a + list of fields to return + :arg _source_exclude: A list of fields to exclude from the returned + _source field + :arg _source_include: A list of fields to extract and return from the + _source field + :arg allow_no_indices: Whether to ignore if a wildcard indices + expression resolves into no concrete indices. (This includes `_all` + string or when no indices have been specified) + :arg allow_partial_search_results: Set to false to return an overall + failure if the request would produce partial results. Defaults to + True, which will allow partial results in the case of timeouts or + partial failures + :arg analyze_wildcard: Specify whether wildcard and prefix queries + should be analyzed (default: false) + :arg analyzer: The analyzer to use for the query string + :arg batched_reduce_size: The number of shard results that should be + reduced at once on the coordinating node. This value should be used + as a protection mechanism to reduce the memory overhead per search + request if the potential number of shards in the request can be + large., default 512 + :arg default_operator: The default operator for query string query (AND + or OR), default 'OR', valid choices are: 'AND', 'OR' + :arg df: The field to use as default where no field prefix is given in + the query string + :arg docvalue_fields: A comma-separated list of fields to return as the + docvalue representation of a field for each hit + :arg expand_wildcards: Whether to expand wildcard expression to concrete + indices that are open, closed or both., default 'open', valid + choices are: 'open', 'closed', 'none', 'all' + :arg explain: Specify whether to return detailed information about score + computation as part of a hit + :arg from\\_: Starting offset (default: 0) + :arg ignore_unavailable: Whether specified concrete indices should be + ignored when unavailable (missing or closed) + :arg lenient: Specify whether format-based query failures (such as + providing text to a numeric field) should be ignored + :arg max_concurrent_shard_requests: The number of concurrent shard + requests this search executes concurrently. This value should be + used to limit the impact of the search on the cluster in order to + limit the number of concurrent shard requests, default 'The default + grows with the number of nodes in the cluster but is at most 256.' + :arg pre_filter_shard_size: A threshold that enforces a pre-filter + roundtrip to prefilter search shards based on query rewriting if + the number of shards the search request expands to exceeds the + threshold. This filter roundtrip can limit the number of shards + significantly if for instance a shard can not match any documents + based on it's rewrite method ie. if date filters are mandatory to + match but the shard bounds and the query are disjoint., default 128 + :arg preference: Specify the node or shard the operation should be + performed on (default: random) + :arg q: Query in the Lucene query string syntax + :arg rest_total_hits_as_int: This parameter is used to restore the total hits as a number + in the response. This param is added version 6.x to handle mixed cluster queries where nodes + are in multiple versions (7.0 and 6.latest) + :arg request_cache: Specify if request cache should be used for this + request or not, defaults to index level setting + :arg routing: A comma-separated list of specific routing values + :arg scroll: Specify how long a consistent view of the index should be + maintained for scrolled search + :arg search_type: Search operation type, valid choices are: + 'query_then_fetch', 'dfs_query_then_fetch' + :arg size: Number of hits to return (default: 10) + :arg sort: A comma-separated list of : pairs + :arg stats: Specific 'tag' of the request for logging and statistical + purposes + :arg stored_fields: A comma-separated list of stored fields to return as + part of a hit + :arg suggest_field: Specify which field to use for suggestions + :arg suggest_mode: Specify suggest mode, default 'missing', valid + choices are: 'missing', 'popular', 'always' + :arg suggest_size: How many suggestions to return in response + :arg suggest_text: The source text for which the suggestions should be + returned + :arg terminate_after: The maximum number of documents to collect for + each shard, upon reaching which the query execution will terminate + early. + :arg timeout: Explicit operation timeout + :arg track_scores: Whether to calculate and return scores even if they + are not used for sorting + :arg track_total_hits: Indicate if the number of documents that match + the query should be tracked + :arg typed_keys: Specify whether aggregation and suggester names should + be prefixed by their respective types in the response + :arg version: Specify whether to return document version as part of a + hit + """ + logging.warning( + 'doc_type has been deprecated since elasticsearch version 6 and will be completely removed in 8') + # from is a reserved word so it cannot be used, use from_ instead + if "from_" in params: + params["from"] = params.pop("from_") + + if not index: + index = "_all" + return self.transport.perform_request( + "GET", _make_path(index, doc_type, "_search"), params=params, body=body + ) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 05296f82e..54ddbddd0 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -59,20 +59,30 @@ def main(es_client, ea_index, recreate=False, old_ea_index=None): # To avoid a race condition. TODO: replace this with a real check time.sleep(2) - if elasticversion > 5: - # TODO remove doc_type for elasticsearch >= 7 when elastic client supports doc_type=None - params = {'include_type_name': 'true'} if elasticversion > 6 else {} - + if elasticversion > 6: + # TODO remove doc_type completely when elasicsearch client allows doc_type=None + # doc_type is a deprecated feature and will be completely removed in Elasicsearch 8 es_client.indices.put_mapping(index=ea_index, doc_type='_doc', - body=es_index_mappings['elastalert'], params=params) + body=es_index_mappings['elastalert'], include_type_name=True) es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc', - body=es_index_mappings['elastalert_status'], params=params) + body=es_index_mappings['elastalert_status'], include_type_name=True) es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc', - body=es_index_mappings['silence'], params=params) + body=es_index_mappings['silence'], include_type_name=True) es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc', - body=es_index_mappings['elastalert_error'], params=params) + body=es_index_mappings['elastalert_error'], include_type_name=True) es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc', - body=es_index_mappings['past_elastalert'], params=params) + body=es_index_mappings['past_elastalert'], include_type_name=True) + elif elasticversion > 5: + es_client.indices.put_mapping(index=ea_index, doc_type='_doc', + body=es_index_mappings['elastalert']) + es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc', + body=es_index_mappings['elastalert_status']) + es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc', + body=es_index_mappings['silence']) + es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc', + body=es_index_mappings['elastalert_error']) + es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc', + body=es_index_mappings['past_elastalert']) else: es_client.indices.put_mapping(index=ea_index, doc_type='elastalert', body=es_index_mappings['elastalert']) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index ec799e258..dbd8ef310 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -266,9 +266,8 @@ def get_index_start(self, index, timestamp_field='@timestamp'): query = {'sort': {timestamp_field: {'order': 'asc'}}} try: if self.current_es.is_atleastsixsix(): - # TODO use _source_includes=[...] instead when elasticsearch client supports this res = self.current_es.search(index=index, size=1, body=query, - params={'_source_includes': timestamp_field}, ignore_unavailable=True) + _source_includes=[timestamp_field], ignore_unavailable=True) else: res = self.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field], ignore_unavailable=True) @@ -343,11 +342,7 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): five=rule['five'], ) if self.current_es.is_atleastsixsix(): - # TODO fix when elasticsearch client supports param _source_includes - # Since _source_includes is not supported we must use params instead. - # the value object in _source_includes is not automagically parsed into a legal - # url query parameter array so we must explicitly handle that as well - extra_args = {'params': {'_source_includes': (','.join(rule['include']))}} + extra_args = {'_source_includes': rule['include']} else: extra_args = {'_source_include': rule['include']} scroll_keepalive = rule.get('scroll_keepalive', self.scroll_keepalive) @@ -492,7 +487,7 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non try: if not rule['five']: - res = self.current_es.search( + res = self.current_es.deprecated_search( index=index, doc_type=rule['doc_type'], body=query, @@ -500,8 +495,8 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule['doc_type'], body=query, size=0, - ignore_unavailable=True) + res = self.current_es.deprecated_search(index=index, doc_type=rule['doc_type'], + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -539,7 +534,7 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ query = self.get_aggregation_query(base_query, rule, query_key, term_size, rule['timestamp_field']) try: if not rule['five']: - res = self.current_es.search( + res = self.current_es.deprecated_search( index=index, doc_type=rule.get('doc_type'), body=query, @@ -547,8 +542,8 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule.get('doc_type'), body=query, size=0, - ignore_unavailable=True) + res = self.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'), + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: if len(str(e)) > 1024: e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024) @@ -662,15 +657,14 @@ def get_starttime(self, rule): if self.writeback_es.is_atleastsix(): index = self.get_six_index('elastalert_status') if self.writeback_es.is_atleastsixsix(): - # TODO use _source_includes=[...] instead when elasticsearch client supports this - res = self.writeback_es.search(index=index, doc_type='_doc', size=1, body=query, - params={'_source_includes': 'endtime,rule_name'}) + res = self.writeback_es.search(index=index, size=1, body=query, + _source_includes=['endtime', 'rule_name']) else: - res = self.writeback_es.search(index=index, doc_type='_doc', size=1, body=query, + res = self.writeback_es.search(index=index, size=1, body=query, _source_include=['endtime', 'rule_name']) else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) + res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert_status', + size=1, body=query, _source_include=['endtime', 'rule_name']) if res['hits']['hits']: endtime = ts_to_dt(res['hits']['hits'][0]['_source']['endtime']) @@ -1310,13 +1304,8 @@ def get_dashboard(self, rule, db_name): raise EAException("use_kibana_dashboard undefined") query = {'query': {'term': {'_id': db_name}}} try: - if es.is_atleastsixsix(): - # TODO use doc_type = _doc - # TODO use _source_includes=[...] instead when elasticsearch client supports for this - res = es.search(index='kibana-int', doc_type='dashboard', body=query, - params={'_source_includes': 'dashboard'}) - else: - res = es.search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) + # TODO use doc_type = _doc + res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) except ElasticsearchException as e: raise EAException("Error querying for dashboard: %s" % (e)), None, sys.exc_info()[2] @@ -1510,7 +1499,7 @@ def writeback(self, doc_type, body): try: if self.writeback_es.is_atleastsix(): writeback_index = self.get_six_index(doc_type) - res = self.writeback_es.index(index=writeback_index, doc_type='_doc', body=body) + res = self.writeback_es.index(index=writeback_index, body=body) else: res = self.writeback_es.index(index=self.writeback_index, doc_type=doc_type, body=body) return res @@ -1536,9 +1525,10 @@ def find_recent_pending_alerts(self, time_limit): query.update(sort) try: if self.writeback_es.is_atleastsix(): - res = self.writeback_es.search(index=self.writeback_index, doc_type='_doc', body=query, size=1000) + res = self.writeback_es.search(index=self.writeback_index, body=query, size=1000) else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert', body=query, size=1000) + res = self.writeback_es.deprecated_search(index=self.writeback_index, + doc_type='elastalert', body=query, size=1000) if res['hits']['hits']: return res['hits']['hits'] except ElasticsearchException as e: @@ -1624,11 +1614,11 @@ def get_aggregated_matches(self, _id): matches = [] try: if self.writeback_es.is_atleastsix(): - res = self.writeback_es.search(index=self.writeback_index, doc_type='_doc', body=query, + res = self.writeback_es.search(index=self.writeback_index, body=query, size=self.max_aggregation) else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert', body=query, - size=self.max_aggregation) + res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert', + body=query, size=self.max_aggregation) for match in res['hits']['hits']: matches.append(match['_source']) if self.writeback_es.is_atleastsix(): @@ -1651,9 +1641,9 @@ def find_pending_aggregate_alert(self, rule, aggregation_key_value=None): query['sort'] = {'alert_time': {'order': 'desc'}} try: if self.writeback_es.is_atleastsix(): - res = self.writeback_es.search(index=self.writeback_index, doc_type='_doc', body=query, size=1) + res = self.writeback_es.search(index=self.writeback_index, body=query, size=1) else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert', body=query, size=1) + res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert', body=query, size=1) if len(res['hits']['hits']) == 0: return None except (KeyError, ElasticsearchException) as e: @@ -1795,15 +1785,14 @@ def is_silenced(self, rule_name): if self.writeback_es.is_atleastsix(): index = self.get_six_index('silence') if self.writeback_es.is_atleastsixsix(): - # TODO use _source_includes=[...] instead when elasticsearch client supports this - res = self.writeback_es.search(index=index, doc_type='_doc', - size=1, body=query, params={'_source_includes': 'until,exponent'}) + res = self.writeback_es.search(index=index, size=1, body=query, + _source_includes=['until', 'exponent']) else: - res = self.writeback_es.search(index=index, doc_type='_doc', - size=1, body=query, _source_include=['until', 'exponent']) + res = self.writeback_es.search(index=index, size=1, body=query, + _source_include=['until', 'exponent']) else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) + res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='silence', + size=1, body=query, _source_include=['until', 'exponent']) except ElasticsearchException as e: self.handle_error("Error while querying for alert silence status: %s" % (e), {'rule': rule_name}) diff --git a/elastalert/rule_from_kibana.py b/elastalert/rule_from_kibana.py index 07310c8dc..ef1392b28 100644 --- a/elastalert/rule_from_kibana.py +++ b/elastalert/rule_from_kibana.py @@ -6,9 +6,9 @@ import json import yaml -from elasticsearch.client import Elasticsearch from elastalert.kibana import filters_from_dashboard +from elastalert.util import elasticsearch_client def main(): @@ -16,20 +16,19 @@ def main(): es_port = raw_input("Elasticsearch port: ") db_name = raw_input("Dashboard name: ") send_get_body_as = raw_input("Method for querying Elasticsearch[GET]: ") or 'GET' - es = Elasticsearch(host=es_host, port=es_port, send_get_body_as=send_get_body_as) - es_version = es.info()["version"]["number"] - print("Elastic Version:" + es_version) + es = elasticsearch_client({'es_host': es_host, 'es_port': es_port, 'send_get_body_as': send_get_body_as}) + + print("Elastic Version:" + es.es_version) query = {'query': {'term': {'_id': db_name}}} - if is_atleastsixsix(es_version): + if es.is_atleastsixsix(): # TODO check support for kibana 7 # TODO use doc_type='_doc' instead - # TODO use _source_includes=[...] instead when elasticsearch client supports this - res = es.search(index='kibana-int', doc_type='dashboard', body=query, params={'_source_includes': 'dashboard'}) + res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_includes=['dashboard']) else: - res = es.search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) + res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) if not res['hits']['hits']: print("No dashboard %s found" % (db_name)) @@ -47,10 +46,5 @@ def main(): print(yaml.safe_dump(config_filters)) -def is_atleastsixsix(es_version): - major, minor = map(int, es_version.split(".")[:2]) - return major > 6 or (major == 6 and minor >= 6) - - if __name__ == '__main__': main() diff --git a/requirements.txt b/requirements.txt index 4f23f2ec1..a15e38eb1 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,7 @@ blist>=1.3.6 boto3>=1.4.4 configparser>=3.5.0 croniter>=0.3.16 -elasticsearch +elasticsearch>=7.0.0 envparse>=0.2.0 exotel>=0.1.3 jira>=1.0.10,<1.0.15 diff --git a/setup.py b/setup.py index 91a514f18..90074821b 100644 --- a/setup.py +++ b/setup.py @@ -32,7 +32,7 @@ 'boto3>=1.4.4', 'configparser>=3.5.0', 'croniter>=0.3.16', - 'elasticsearch', + 'elasticsearch>=7.0.0', 'envparse>=0.2.0', 'exotel>=0.1.3', 'jira>=1.0.10,<1.0.15', diff --git a/tests/base_test.py b/tests/base_test.py index 559ed1f85..48f5d6dfc 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -103,7 +103,7 @@ def test_query_sixsix(ea_sixsix): ea_sixsix.current_es.search.assert_called_with(body={ 'query': {'bool': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', params={'_source_includes': '@timestamp'}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_includes=['@timestamp'], ignore_unavailable=True, size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) @@ -156,7 +156,7 @@ def test_query_sixsix_with_unix(ea_sixsix): ea_sixsix.current_es.search.assert_called_with( body={'query': {'bool': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', params={'_source_includes': '@timestamp'}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_includes=['@timestamp'], ignore_unavailable=True, size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) @@ -186,7 +186,7 @@ def test_query_sixsix_with_unixms(ea_sixsix): ea_sixsix.current_es.search.assert_called_with( body={'query': {'bool': { 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', params={'_source_includes': '@timestamp'}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_includes=['@timestamp'], ignore_unavailable=True, size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) @@ -201,7 +201,7 @@ def test_no_terms_hits(ea): ea.rules[0]['use_terms_query'] = True ea.rules[0]['query_key'] = 'QWERTY' ea.rules[0]['doc_type'] = 'uiop' - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.current_es.deprecated_search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_terms_data.call_count == 0 @@ -318,16 +318,16 @@ def test_match_with_module_from_pending(ea): pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} # First call, return the pending alert, second, no associated aggregated alerts - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, - {'hits': {'hits': []}}] + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + {'hits': {'hits': []}}] ea.send_pending_alerts() assert mod.process.call_count == 0 # If aggregation is set, enhancement IS called pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, - {'hits': {'hits': []}}] + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, + {'hits': {'hits': []}}] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.send_pending_alerts() assert mod.process.call_count == 1 @@ -377,8 +377,9 @@ def test_agg_matchtime(ea): alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10)) hits = generate_hits(hits_timestamps) ea.current_es.search.return_value = hits - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: # Aggregate first two, query over full range + mock_es.return_value = ea.current_es ea.rules[0]['aggregate_by_match_time'] = True ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps] @@ -404,10 +405,10 @@ def test_agg_matchtime(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call3}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}}, - {'hits': {'total': 0, 'hits': []}}] + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, + {'_id': 'CDEF', '_source': call3}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}}, + {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: ea.send_pending_alerts() @@ -416,15 +417,15 @@ def test_agg_matchtime(ea): assert mock_es.call_count == 2 assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]]) - call1 = ea.writeback_es.search.call_args_list[7][1]['body'] - call2 = ea.writeback_es.search.call_args_list[8][1]['body'] - call3 = ea.writeback_es.search.call_args_list[9][1]['body'] - call4 = ea.writeback_es.search.call_args_list[10][1]['body'] + call1 = ea.writeback_es.deprecated_search.call_args_list[7][1]['body'] + call2 = ea.writeback_es.deprecated_search.call_args_list[8][1]['body'] + call3 = ea.writeback_es.deprecated_search.call_args_list[9][1]['body'] + call4 = ea.writeback_es.deprecated_search.call_args_list[10][1]['body'] assert 'alert_time' in call2['filter']['range'] assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD' assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF' - assert ea.writeback_es.search.call_args_list[9][1]['size'] == 1337 + assert ea.writeback_es.deprecated_search.call_args_list[9][1]['size'] == 1337 def test_agg_not_matchtime(ea): @@ -530,7 +531,8 @@ def test_agg_with_aggregation_key(ea): match_time = ts_to_dt('2014-09-26T12:45:00Z') hits = generate_hits(hits_timestamps) ea.current_es.search.return_value = hits - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: + mock_es.return_value = ea.current_es with mock.patch('elastalert.elastalert.ts_now', return_value=match_time): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps] @@ -571,27 +573,28 @@ def test_agg_with_aggregation_key(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call2}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}}, - {'hits': {'total': 0, 'hits': []}}] + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, + {'_id': 'CDEF', '_source': call2}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}}, + {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: + mock_es.return_value = ea.current_es ea.send_pending_alerts() # Assert that current_es was refreshed from the aggregate rules assert mock_es.called_with(host='', port='') assert mock_es.call_count == 2 assert_alerts(ea, [[hits_timestamps[0], hits_timestamps[2]], [hits_timestamps[1]]]) - call1 = ea.writeback_es.search.call_args_list[7][1]['body'] - call2 = ea.writeback_es.search.call_args_list[8][1]['body'] - call3 = ea.writeback_es.search.call_args_list[9][1]['body'] - call4 = ea.writeback_es.search.call_args_list[10][1]['body'] + call1 = ea.writeback_es.deprecated_search.call_args_list[7][1]['body'] + call2 = ea.writeback_es.deprecated_search.call_args_list[8][1]['body'] + call3 = ea.writeback_es.deprecated_search.call_args_list[9][1]['body'] + call4 = ea.writeback_es.deprecated_search.call_args_list[10][1]['body'] assert 'alert_time' in call2['filter']['range'] assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD' assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF' - assert ea.writeback_es.search.call_args_list[9][1]['size'] == 1337 + assert ea.writeback_es.deprecated_search.call_args_list[9][1]['size'] == 1337 def test_silence(ea): @@ -944,15 +947,15 @@ def test_kibana_dashboard(ea): mock_es_init.return_value = mock_es # No dashboard found - mock_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + mock_es.deprecated_search.return_value = {'hits': {'total': 0, 'hits': []}} with pytest.raises(EAException): ea.use_kibana_link(ea.rules[0], match) - mock_call = mock_es.search.call_args_list[0][1] + mock_call = mock_es.deprecated_search.call_args_list[0][1] assert mock_call['body'] == {'query': {'term': {'_id': 'my dashboard'}}} # Dashboard found mock_es.index.return_value = {'_id': 'ABCDEFG'} - mock_es.search.return_value = {'hits': {'hits': [{'_source': {'dashboard': json.dumps(dashboard_temp)}}]}} + mock_es.deprecated_search.return_value = {'hits': {'hits': [{'_source': {'dashboard': json.dumps(dashboard_temp)}}]}} url = ea.use_kibana_link(ea.rules[0], match) assert 'ABCDEFG' in url db = json.loads(mock_es.index.call_args_list[0][1]['body']['dashboard']) @@ -1077,9 +1080,9 @@ def test_count_keys(ea): 'filtered': {'counts': {'buckets': [{'key': 'a', 'doc_count': 10}, {'key': 'b', 'doc_count': 5}]}}}}, {'aggregations': {'filtered': { 'counts': {'buckets': [{'key': 'd', 'doc_count': 10}, {'key': 'c', 'doc_count': 12}]}}}}] - ea.current_es.search.side_effect = buckets + ea.current_es.deprecated_search.side_effect = buckets counts = ea.get_top_counts(ea.rules[0], START, END, ['this', 'that']) - calls = ea.current_es.search.call_args_list + calls = ea.current_es.deprecated_search.call_args_list assert calls[0][1]['search_type'] == 'count' assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5, 'min_doc_count': 1} diff --git a/tests/conftest.py b/tests/conftest.py index 2f91aa5a7..6b16e3994 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -61,28 +61,18 @@ def __init__(self, host='es', port=14900): self.port = port self.return_hits = [] self.search = mock.Mock() + self.deprecated_search = mock.Mock() self.create = mock.Mock() self.index = mock.Mock() self.delete = mock.Mock() self.info = mock.Mock(return_value={'status': 200, 'name': 'foo', 'version': {'number': '2.0'}}) self.ping = mock.Mock(return_value=True) self.indices = mock_es_indices_client() - - @property - def es_version(self): - return self.info()['version']['number'] - - def is_atleastfive(self): - return False - - def is_atleastsix(self): - return False - - def is_atleastsixsix(self): - return False - - def is_atleastseven(self): - return False + self.es_version = mock.Mock(return_value='2.0') + self.is_atleastfive = mock.Mock(return_value=False) + self.is_atleastsix = mock.Mock(return_value=False) + self.is_atleastsixsix = mock.Mock(return_value=False) + self.is_atleastseven = mock.Mock(return_value=False) class mock_es_sixsix_client(object): @@ -91,28 +81,18 @@ def __init__(self, host='es', port=14900): self.port = port self.return_hits = [] self.search = mock.Mock() + self.deprecated_search = mock.Mock() self.create = mock.Mock() self.index = mock.Mock() self.delete = mock.Mock() self.info = mock.Mock(return_value={'status': 200, 'name': 'foo', 'version': {'number': '6.6.0'}}) self.ping = mock.Mock(return_value=True) self.indices = mock_es_indices_client() - - @property - def es_version(self): - return self.info()['version']['number'] - - def is_atleastfive(self): - return True - - def is_atleastsix(self): - return True - - def is_atleastsixsix(self): - return True - - def is_atleastseven(self): - return False + self.es_version = mock.Mock(return_value='6.6.0') + self.is_atleastfive = mock.Mock(return_value=True) + self.is_atleastsix = mock.Mock(return_value=True) + self.is_atleastsixsix = mock.Mock(return_value=True) + self.is_atleastseven = mock.Mock(return_value=False) class mock_ruletype(object): @@ -164,6 +144,7 @@ def ea(): 'old_query_limit': datetime.timedelta(weeks=1), 'disable_rules_on_error': False, 'scroll_keepalive': '30s'} + elastalert.util.elasticsearch_client = mock_es_client elastalert.elastalert.elasticsearch_client = mock_es_client with mock.patch('elastalert.elastalert.get_rule_hashes'): with mock.patch('elastalert.elastalert.load_rules') as load_conf: @@ -173,6 +154,7 @@ def ea(): ea.rules[0]['alert'] = [mock_alert()] ea.writeback_es = mock_es_client() ea.writeback_es.search.return_value = {'hits': {'hits': []}} + ea.writeback_es.deprecated_search.return_value = {'hits': {'hits': []}} ea.writeback_es.index.return_value = {'_id': 'ABCD'} ea.current_es = mock_es_client('', '') return ea @@ -209,6 +191,7 @@ def ea_sixsix(): 'disable_rules_on_error': False, 'scroll_keepalive': '30s'} elastalert.elastalert.elasticsearch_client = mock_es_sixsix_client + elastalert.util.elasticsearch_client = mock_es_sixsix_client with mock.patch('elastalert.elastalert.get_rule_hashes'): with mock.patch('elastalert.elastalert.load_rules') as load_conf: load_conf.return_value = conf @@ -217,6 +200,7 @@ def ea_sixsix(): ea_sixsix.rules[0]['alert'] = [mock_alert()] ea_sixsix.writeback_es = mock_es_sixsix_client() ea_sixsix.writeback_es.search.return_value = {'hits': {'hits': []}} + ea_sixsix.writeback_es.deprecated_search.return_value = {'hits': {'hits': []}} ea_sixsix.writeback_es.index.return_value = {'_id': 'ABCD'} ea_sixsix.current_es = mock_es_sixsix_client('', -1) return ea_sixsix From 2434ef623dadf98dea9845e997366e9eafe4fe57 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Fri, 12 Apr 2019 13:54:52 +0200 Subject: [PATCH 175/264] Elasticsearch did not allow 'doc_type' starting with _ until version 6.2. This is now reflected in ElastAlert --- .travis.yml | 1 + elastalert/__init__.py | 16 ++++++++++++ elastalert/create_index.py | 33 ++++++++++++++++++++---- elastalert/elastalert.py | 51 ++++++++++++++------------------------ tests/conftest.py | 22 ++++++++++++++-- 5 files changed, 84 insertions(+), 39 deletions(-) diff --git a/.travis.yml b/.travis.yml index c3a8b6a37..7c8ba2b34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -26,6 +26,7 @@ jobs: env: TOXENV=py27 ES_VERSION=7.0.0-linux-x86_64 - env: TOXENV=py27 ES_VERSION=6.6.2 - env: TOXENV=py27 ES_VERSION=6.3.2 + - env: TOXENV=py27 ES_VERSION=6.2.4 - env: TOXENV=py27 ES_VERSION=6.0.1 - env: TOXENV=py27 ES_VERSION=5.6.16 diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 7716c253a..90b14126e 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -75,6 +75,22 @@ def is_atleastseven(self): """ return int(self.es_version.split(".")[0]) >= 7 + def resolve_writeback_index(self, writeback_index, doc_type): + """ In ES6, you cannot have multiple _types per index, + therefore we use self.writeback_index as the prefix for the actual + index name, based on doc_type. """ + if not self.is_atleastsix(): + return writeback_index + elif doc_type == 'silence': + return writeback_index + '_silence' + elif doc_type == 'past_elastalert': + return writeback_index + '_past' + elif doc_type == 'elastalert_status': + return writeback_index + '_status' + elif doc_type == 'elastalert_error': + return writeback_index + '_error' + return writeback_index + @query_params( "_source", "_source_exclude", diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 54ddbddd0..41def2cbd 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -23,9 +23,8 @@ def main(es_client, ea_index, recreate=False, old_ea_index=None): esversion = es_client.info()["version"]["number"] print("Elastic Version: " + esversion) - elasticversion = int(esversion.split(".")[0]) - es_index_mappings = read_es_index_mappings() if elasticversion > 5 else read_es_index_mappings(5) + es_index_mappings = read_es_index_mappings() if is_atleastsix(esversion) else read_es_index_mappings(5) es_index = IndicesClient(es_client) if not recreate: @@ -34,7 +33,7 @@ def main(es_client, ea_index, recreate=False, old_ea_index=None): return None # (Re-)Create indices. - if (elasticversion > 5): + if is_atleastsix(esversion): index_names = ( ea_index, ea_index + '_status', @@ -59,7 +58,7 @@ def main(es_client, ea_index, recreate=False, old_ea_index=None): # To avoid a race condition. TODO: replace this with a real check time.sleep(2) - if elasticversion > 6: + if is_atleastseven(esversion): # TODO remove doc_type completely when elasicsearch client allows doc_type=None # doc_type is a deprecated feature and will be completely removed in Elasicsearch 8 es_client.indices.put_mapping(index=ea_index, doc_type='_doc', @@ -72,7 +71,7 @@ def main(es_client, ea_index, recreate=False, old_ea_index=None): body=es_index_mappings['elastalert_error'], include_type_name=True) es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc', body=es_index_mappings['past_elastalert'], include_type_name=True) - elif elasticversion > 5: + elif is_atleastsixtwo(esversion): es_client.indices.put_mapping(index=ea_index, doc_type='_doc', body=es_index_mappings['elastalert']) es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc', @@ -83,6 +82,17 @@ def main(es_client, ea_index, recreate=False, old_ea_index=None): body=es_index_mappings['elastalert_error']) es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc', body=es_index_mappings['past_elastalert']) + elif is_atleastsix(esversion): + es_client.indices.put_mapping(index=ea_index, doc_type='elastalert', + body=es_index_mappings['elastalert']) + es_client.indices.put_mapping(index=ea_index + '_status', doc_type='elastalert_status', + body=es_index_mappings['elastalert_status']) + es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='silence', + body=es_index_mappings['silence']) + es_client.indices.put_mapping(index=ea_index + '_error', doc_type='elastalert_error', + body=es_index_mappings['elastalert_error']) + es_client.indices.put_mapping(index=ea_index + '_past', doc_type='past_elastalert', + body=es_index_mappings['past_elastalert']) else: es_client.indices.put_mapping(index=ea_index, doc_type='elastalert', body=es_index_mappings['elastalert']) @@ -124,6 +134,19 @@ def read_es_index_mapping(mapping, es_version=6): return json.load(f) +def is_atleastsix(es_version): + return int(es_version.split(".")[0]) >= 6 + + +def is_atleastsixtwo(es_version): + major, minor = map(int, es_version.split(".")[:2]) + return major > 6 or (major == 6 and minor >= 2) + + +def is_atleastseven(es_version): + return int(es_version.split(".")[0]) >= 7 + + if __name__ == '__main__': parser = argparse.ArgumentParser() parser.add_argument('--host', default=os.environ.get('ES_HOST', None), help='Elasticsearch host') diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index dbd8ef310..a0e84fa09 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -162,21 +162,6 @@ def get_index(rule, starttime=None, endtime=None): else: return index - def get_six_index(self, doc_type): - """ In ES6, you cannot have multiple _types per index, - therefore we use self.writeback_index as the prefix for the actual - index name, based on doc_type. """ - writeback_index = self.writeback_index - if doc_type == 'silence': - writeback_index += '_silence' - elif doc_type == 'past_elastalert': - writeback_index += '_past' - elif doc_type == 'elastalert_status': - writeback_index += '_status' - elif doc_type == 'elastalert_error': - writeback_index += '_error' - return writeback_index - @staticmethod def get_query(filters, starttime=None, endtime=None, sort=True, timestamp_field='@timestamp', to_ts_func=dt_to_ts, desc=False, five=False): @@ -654,8 +639,9 @@ def get_starttime(self, rule): query.update(sort) try: - if self.writeback_es.is_atleastsix(): - index = self.get_six_index('elastalert_status') + doc_type = 'elastalert_status' + index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type) + if self.writeback_es.is_atleastsixtwo(): if self.writeback_es.is_atleastsixsix(): res = self.writeback_es.search(index=index, size=1, body=query, _source_includes=['endtime', 'rule_name']) @@ -663,7 +649,7 @@ def get_starttime(self, rule): res = self.writeback_es.search(index=index, size=1, body=query, _source_include=['endtime', 'rule_name']) else: - res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert_status', + res = self.writeback_es.deprecated_search(index=index, doc_type=doc_type, size=1, body=query, _source_include=['endtime', 'rule_name']) if res['hits']['hits']: endtime = ts_to_dt(res['hits']['hits'][0]['_source']['endtime']) @@ -1497,11 +1483,11 @@ def writeback(self, doc_type, body): writeback_body['@timestamp'] = dt_to_ts(ts_now()) try: - if self.writeback_es.is_atleastsix(): - writeback_index = self.get_six_index(doc_type) - res = self.writeback_es.index(index=writeback_index, body=body) + index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type) + if self.writeback_es.is_atleastsixtwo(): + res = self.writeback_es.index(index=index, body=body) else: - res = self.writeback_es.index(index=self.writeback_index, doc_type=doc_type, body=body) + res = self.writeback_es.index(index=index, doc_type=doc_type, body=body) return res except ElasticsearchException as e: logging.exception("Error writing alert info to Elasticsearch: %s" % (e)) @@ -1524,7 +1510,7 @@ def find_recent_pending_alerts(self, time_limit): query = {'query': inner_query, 'filter': time_filter} query.update(sort) try: - if self.writeback_es.is_atleastsix(): + if self.writeback_es.is_atleastsixtwo(): res = self.writeback_es.search(index=self.writeback_index, body=query, size=1000) else: res = self.writeback_es.deprecated_search(index=self.writeback_index, @@ -1580,8 +1566,8 @@ def send_pending_alerts(self): # Delete it from the index try: - if self.writeback_es.is_atleastsix(): - self.writeback_es.delete(index=self.writeback_index, doc_type='_doc', id=_id) + if self.writeback_es.is_atleastsixtwo(): + self.writeback_es.delete(index=self.writeback_index, id=_id) else: self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=_id) except ElasticsearchException: # TODO: Give this a more relevant exception, try:except: is evil. @@ -1613,7 +1599,7 @@ def get_aggregated_matches(self, _id): query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}} matches = [] try: - if self.writeback_es.is_atleastsix(): + if self.writeback_es.is_atleastsixtwo(): res = self.writeback_es.search(index=self.writeback_index, body=query, size=self.max_aggregation) else: @@ -1621,8 +1607,8 @@ def get_aggregated_matches(self, _id): body=query, size=self.max_aggregation) for match in res['hits']['hits']: matches.append(match['_source']) - if self.writeback_es.is_atleastsix(): - self.writeback_es.delete(index=self.writeback_index, doc_type='_doc', id=match['_id']) + if self.writeback_es.is_atleastsixtwo(): + self.writeback_es.delete(index=self.writeback_index, id=match['_id']) else: self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=match['_id']) except (KeyError, ElasticsearchException) as e: @@ -1640,7 +1626,7 @@ def find_pending_aggregate_alert(self, rule, aggregation_key_value=None): query = {'query': {'bool': query}} query['sort'] = {'alert_time': {'order': 'desc'}} try: - if self.writeback_es.is_atleastsix(): + if self.writeback_es.is_atleastsixtwo(): res = self.writeback_es.search(index=self.writeback_index, body=query, size=1) else: res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert', body=query, size=1) @@ -1782,8 +1768,9 @@ def is_silenced(self, rule_name): query.update(sort) try: - if self.writeback_es.is_atleastsix(): - index = self.get_six_index('silence') + doc_type = 'silence' + index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type) + if self.writeback_es.is_atleastsixtwo(): if self.writeback_es.is_atleastsixsix(): res = self.writeback_es.search(index=index, size=1, body=query, _source_includes=['until', 'exponent']) @@ -1791,7 +1778,7 @@ def is_silenced(self, rule_name): res = self.writeback_es.search(index=index, size=1, body=query, _source_include=['until', 'exponent']) else: - res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='silence', + res = self.writeback_es.deprecated_search(index=index, doc_type=doc_type, size=1, body=query, _source_include=['until', 'exponent']) except ElasticsearchException as e: self.handle_error("Error while querying for alert silence status: %s" % (e), {'rule': rule_name}) diff --git a/tests/conftest.py b/tests/conftest.py index 6b16e3994..6b9db2894 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -11,6 +11,8 @@ from elastalert.util import dt_to_ts from elastalert.util import ts_to_dt +writeback_index = 'wb' + def pytest_addoption(parser): parser.addoption( @@ -71,8 +73,10 @@ def __init__(self, host='es', port=14900): self.es_version = mock.Mock(return_value='2.0') self.is_atleastfive = mock.Mock(return_value=False) self.is_atleastsix = mock.Mock(return_value=False) + self.is_atleastsixtwo = mock.Mock(return_value=False) self.is_atleastsixsix = mock.Mock(return_value=False) self.is_atleastseven = mock.Mock(return_value=False) + self.resolve_writeback_index = mock.Mock(return_value=writeback_index) class mock_es_sixsix_client(object): @@ -91,9 +95,23 @@ def __init__(self, host='es', port=14900): self.es_version = mock.Mock(return_value='6.6.0') self.is_atleastfive = mock.Mock(return_value=True) self.is_atleastsix = mock.Mock(return_value=True) + self.is_atleastsixtwo = mock.Mock(return_value=False) self.is_atleastsixsix = mock.Mock(return_value=True) self.is_atleastseven = mock.Mock(return_value=False) + def writeback_index_side_effect(index, doc_type): + if doc_type == 'silence': + return index + '_silence' + elif doc_type == 'past_elastalert': + return index + '_past' + elif doc_type == 'elastalert_status': + return index + '_status' + elif doc_type == 'elastalert_error': + return index + '_error' + return index + + self.resolve_writeback_index = mock.Mock(side_effect=writeback_index_side_effect) + class mock_ruletype(object): def __init__(self): @@ -138,7 +156,7 @@ def ea(): 'alert_time_limit': datetime.timedelta(hours=24), 'es_host': 'es', 'es_port': 14900, - 'writeback_index': 'wb', + 'writeback_index': writeback_index, 'rules': rules, 'max_query_size': 10000, 'old_query_limit': datetime.timedelta(weeks=1), @@ -184,7 +202,7 @@ def ea_sixsix(): 'alert_time_limit': datetime.timedelta(hours=24), 'es_host': 'es', 'es_port': 14900, - 'writeback_index': 'wb', + 'writeback_index': writeback_index, 'rules': rules, 'max_query_size': 10000, 'old_query_limit': datetime.timedelta(weeks=1), From ad4aef3e09d2e3421a2ad646c4ae022cbf1226fa Mon Sep 17 00:00:00 2001 From: Nenad Merdanovic Date: Wed, 17 Apr 2019 19:07:47 +0200 Subject: [PATCH 176/264] Fix an issue with nested timestamp in cardinality rules (again) --- elastalert/ruletypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index e25ef148b..a8845ca92 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -955,8 +955,8 @@ def garbage_collect(self, timestamp): def get_match_str(self, match): lt = self.rules.get('use_local_time') - starttime = pretty_ts(dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules['timeframe']), lt) - endtime = pretty_ts(match[self.ts_field], lt) + starttime = pretty_ts(dt_to_ts(ts_to_dt(lookup_es_key(match, self.ts_field)) - self.rules['timeframe']), lt) + endtime = pretty_ts(lookup_es_key(match, self.ts_field), lt) if 'max_cardinality' in self.rules: message = ('A maximum of %d unique %s(s) occurred since last alert or between %s and %s\n\n' % (self.rules['max_cardinality'], self.rules['cardinality_field'], From 2647c7de8e52f523948c19e866c1242301a05f3b Mon Sep 17 00:00:00 2001 From: Xuing Date: Fri, 19 Apr 2019 10:59:36 +0800 Subject: [PATCH 177/264] Update config.py Fix YAMLLoadWarning --- elastalert/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/config.py b/elastalert/config.py index af72e5728..b41730108 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -28,7 +28,7 @@ from util import unixms_to_dt # schema for rule yaml -rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) +rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader)) # Required global (config.yaml) and local (rule.yaml) configuration options required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) From 435acbf683dbb4b0bd79a122030a8c9d2cf05c54 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Thu, 25 Apr 2019 07:36:28 +0200 Subject: [PATCH 178/264] Cast --timeout argument to int in elastalert-create-index. fixes #2225 --- elastalert/create_index.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 41def2cbd..ace664784 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -177,7 +177,7 @@ def is_atleastseven(es_version): '--aws-region', default=None, help='AWS Region to use for signing requests. Optionally use the AWS_DEFAULT_REGION environment variable') - parser.add_argument('--timeout', default=60, help='Elasticsearch request timeout') + parser.add_argument('--timeout', default=60, type=int, help='Elasticsearch request timeout') parser.add_argument('--config', default='config.yaml', help='Global config file (default: config.yaml)') parser.add_argument('--recreate', type=bool, default=False, help='Force re-creation of the index (this will cause data loss).') From d81dd93e421e25d1fb18e6d750a1b90f93268e49 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mats-G=C3=B8ran=20Karlsen?= Date: Thu, 25 Apr 2019 14:01:14 +0200 Subject: [PATCH 179/264] Bugfix elastalert-create-index console script Change elastalert.create_index:main signature in order to satisfy entry point constraint in setup tools console script generator --- elastalert/create_index.py | 10 +++++++--- setup.py | 2 +- tests/elasticsearch_test.py | 2 +- 3 files changed, 9 insertions(+), 5 deletions(-) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index ace664784..a6831263c 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -20,7 +20,7 @@ env = Env(ES_USE_SSL=bool) -def main(es_client, ea_index, recreate=False, old_ea_index=None): +def create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None): esversion = es_client.info()["version"]["number"] print("Elastic Version: " + esversion) @@ -147,7 +147,7 @@ def is_atleastseven(es_version): return int(es_version.split(".")[0]) >= 7 -if __name__ == '__main__': +def main(): parser = argparse.ArgumentParser() parser.add_argument('--host', default=os.environ.get('ES_HOST', None), help='Elasticsearch host') parser.add_argument('--port', default=os.environ.get('ES_PORT', None), type=int, help='Elasticsearch port') @@ -255,4 +255,8 @@ def is_atleastseven(es_version): ca_certs=ca_certs, client_key=client_key) - main(es_client=es, ea_index=index, recreate=args.recreate, old_ea_index=old_index) + create_index_mappings(es_client=es, ea_index=index, recreate=args.recreate, old_ea_index=old_index) + + +if __name__ == '__main__': + main() diff --git a/setup.py b/setup.py index 90074821b..d7927eeb2 100644 --- a/setup.py +++ b/setup.py @@ -25,7 +25,7 @@ 'elastalert-rule-from-kibana=elastalert.rule_from_kibana:main', 'elastalert=elastalert.elastalert:main']}, packages=find_packages(), - package_data={'elastalert': ['schema.yaml']}, + package_data={'elastalert': ['schema.yaml', 'es_mappings/**/*.json']}, install_requires=[ 'aws-requests-auth>=0.3.0', 'blist>=1.3.6', diff --git a/tests/elasticsearch_test.py b/tests/elasticsearch_test.py index c293fdf05..b867d3543 100644 --- a/tests/elasticsearch_test.py +++ b/tests/elasticsearch_test.py @@ -76,7 +76,7 @@ class TestElasticsearch(object): # This is not a problem as long as the data is manually removed or the test environment # is torn down after the test run(eg. running tests in a test environment such as Travis) def test_create_indices(self, es_client): - elastalert.create_index.main(es_client=es_client, ea_index=test_index) + elastalert.create_index.create_index_mappings(es_client=es_client, ea_index=test_index) indices_mappings = es_client.indices.get_mapping(test_index + '*') print('-' * 50) print(json.dumps(indices_mappings, indent=2)) From e718988c0a8f92ea3819155ba4bf32e7269358fe Mon Sep 17 00:00:00 2001 From: "karoly.bujtor" Date: Mon, 29 Apr 2019 13:59:48 +0200 Subject: [PATCH 180/264] Add `max_scrolling_count` setting to limit scrolling behavior --- docs/source/elastalert.rst | 6 +++++- elastalert/config.py | 1 + elastalert/elastalert.py | 5 ++++- elastalert/schema.yaml | 1 + elastalert/util.py | 13 +++++++++++++ tests/util_test.py | 15 +++++++++++++++ 6 files changed, 39 insertions(+), 2 deletions(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index ed0877a1c..55aa0057b 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -148,7 +148,11 @@ configuration. ``max_query_size``: The maximum number of documents that will be downloaded from Elasticsearch in a single query. The default is 10,000, and if you expect to get near this number, consider using ``use_count_query`` for the rule. If this -limit is reached, ElastAlert will `scroll `_ through pages the size of ``max_query_size`` until processing all results. +limit is reached, ElastAlert will `scroll `_ +using the size of ``max_query_size`` through the set amount of pages, when ``max_scrolling_count`` is set or until processing all results. + +``max_scrolling_count``: The maximum amount of pages to scroll through. The default is ``0``, which means the scrolling has no limit. +For example if this value is set to ``5`` and the ``max_query_size`` is set to ``10000`` then ``50000`` documents will be downloaded at most. ``scroll_keepalive``: The maximum time (formatted in `Time Units `_) the scrolling context should be kept alive. Avoid using high values as it abuses resources in Elasticsearch, but be mindful to allow sufficient time to finish processing all the results. diff --git a/elastalert/config.py b/elastalert/config.py index af72e5728..31f332344 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -475,6 +475,7 @@ def load_rules(args): conf.setdefault('max_query_size', 10000) conf.setdefault('scroll_keepalive', '30s') + conf.setdefault('max_scrolling_count', 0) conf.setdefault('disable_rules_on_error', True) conf.setdefault('scan_subdirectories', True) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 22b9d49c8..daf468680 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -48,6 +48,7 @@ from util import ts_now from util import ts_to_dt from util import unix_to_dt +from util import should_scrolling_continue class ElastAlerter(): @@ -601,6 +602,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): # Reset hit counter and query rule_inst = rule['type'] + rule['scrolling_cycle'] = rule.get('scrolling_cycle', 0) + 1 index = self.get_index(rule, start, end) if rule.get('use_count_query'): data = self.get_hits_count(rule, start, end, index) @@ -629,7 +631,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): rule_inst.add_data(data) try: - if rule.get('scroll_id') and self.num_hits < self.total_hits: + if rule.get('scroll_id') and self.num_hits < self.total_hits and should_scrolling_continue(rule): self.run_query(rule, start, end, scroll=True) except RuntimeError: # It's possible to scroll far enough to hit max recursive depth @@ -838,6 +840,7 @@ def run_rule(self, rule, endtime, starttime=None): self.set_starttime(rule, endtime) rule['original_starttime'] = rule['starttime'] + rule['scrolling_cycle'] = 0 # Don't run if starttime was set to the future if ts_now() <= rule['starttime']: diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index e5caacc6e..8a3919f76 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -178,6 +178,7 @@ properties: buffer_time: *timeframe query_delay: *timeframe max_query_size: {type: integer} + max_scrolling: {type: integer} owner: {type: string} priority: {type: integer} diff --git a/elastalert/util.py b/elastalert/util.py index d24256994..5a93e7ee3 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -434,3 +434,16 @@ def resolve_string(string, match, missing_text=''): string = string.replace('{%s}' % e.message, '{_missing_value}') return string + + +def should_scrolling_continue(rule_conf): + """ + Tells about a rule config if it can scroll still or should stop the scrolling. + + :param: rule_conf as dict + :rtype: bool + """ + max_scrolling = rule_conf.get('max_scrolling_count') + stop_the_scroll = 0 < max_scrolling <= rule_conf.get('scrolling_cycle') + + return not stop_the_scroll diff --git a/tests/util_test.py b/tests/util_test.py index 3e2a8cbb3..55a2f9c8f 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -14,6 +14,7 @@ from elastalert.util import replace_dots_in_field_names from elastalert.util import resolve_string from elastalert.util import set_es_key +from elastalert.util import should_scrolling_continue @pytest.mark.parametrize('spec, expected_delta', [ @@ -213,3 +214,17 @@ def test_format_index(): 'logstash-2018.06.25', 'logstash-2018.06.26'] assert sorted(format_index(pattern2, date, date2, True).split(',')) == ['logstash-2018.25', 'logstash-2018.26'] + + +def test_should_scrolling_continue(): + rule_no_max_scrolling = {'max_scrolling_count': 0, 'scrolling_cycle': 1} + rule_reached_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 2} + rule_before_first_run = {'max_scrolling_count': 0, 'scrolling_cycle': 0} + rule_before_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 1} + rule_over_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 3} + + assert should_scrolling_continue(rule_no_max_scrolling) is True + assert should_scrolling_continue(rule_reached_max_scrolling) is False + assert should_scrolling_continue(rule_before_first_run) is True + assert should_scrolling_continue(rule_before_max_scrolling) is True + assert should_scrolling_continue(rule_over_max_scrolling) is False From 2734c2eae6c6acd97784746b6870cb4007b5caeb Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 3 May 2019 11:47:40 -0700 Subject: [PATCH 181/264] Added a note on ES7 to the readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 1bfa1f2ea..8cb928155 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,6 @@ +Note: If you're using Elasticsearch 7, you'll need to install a beta release of Elastalert: `pip install "elastalert>=0.2.0b"` + + [![Stories in Ready](https://badge.waffle.io/Yelp/elastalert.png?label=ready&title=Ready)](https://waffle.io/Yelp/elastalert) [![Stories in In Progress](https://badge.waffle.io/Yelp/elastalert.png?label=in%20progress&title=In%20Progress)](https://waffle.io/Yelp/elastalert) [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) From 906d71164395a9801a2c1bc795149602cf9176bc Mon Sep 17 00:00:00 2001 From: 0xflotus <0xflotus@gmail.com> Date: Sat, 4 May 2019 16:15:31 +0200 Subject: [PATCH 182/264] Update ruletypes.rst --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index fa6875aff..0d6bdc3de 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1869,7 +1869,7 @@ The alerter requires the following options: Optional: -``victorops_entity_id``: The identity of the incident used by VictorOps to correlate incidents thoughout the alert lifecycle. If not defined, VictorOps will assign a random string to each alert. +``victorops_entity_id``: The identity of the incident used by VictorOps to correlate incidents throughout the alert lifecycle. If not defined, VictorOps will assign a random string to each alert. ``victorops_entity_display_name``: Human-readable name of alerting entity to summarize incidents without affecting the life-cycle workflow. From 483b16ffc158f1ef36c49f06c06eb9ecaafcf565 Mon Sep 17 00:00:00 2001 From: Caleb Collins-Parks <46505081+caleb15@users.noreply.github.com> Date: Mon, 6 May 2019 15:48:10 -0700 Subject: [PATCH 183/264] remove deprecated option from guide see https://github.com/Yelp/elastalert/issues/633 --- docs/source/recipes/writing_filters.rst | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/docs/source/recipes/writing_filters.rst b/docs/source/recipes/writing_filters.rst index 6a2ca3f07..e923b89a5 100644 --- a/docs/source/recipes/writing_filters.rst +++ b/docs/source/recipes/writing_filters.rst @@ -56,22 +56,23 @@ Note that a term query may not behave as expected if a field is analyzed. By def a field that appears to have the value "foo bar", unless it is not analyzed. Conversely, a term query for "foo" will match analyzed strings "foo bar" and "foo baz". For full text matching on analyzed fields, use query_string. See https://www.elastic.co/guide/en/elasticsearch/guide/current/term-vs-full-text.html -terms +`terms `_ ***** + + Terms allows for easy combination of multiple term filters:: filter: - terms: - field: ["value1", "value2"] + field: ["value1", "value2"] # value1 OR value2 -Using the minimum_should_match option, you can define a set of term filters of which a certain number must match:: +You can also match on multiple fields:: - terms: fieldX: ["value1", "value2"] fieldY: ["something", "something_else"] fieldZ: ["foo", "bar", "baz"] - minimum_should_match: 2 wildcard ******** From 0d92b9b09933ac1bff8bdb81baa46b662f5d3e6b Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Tue, 21 May 2019 13:13:47 +0200 Subject: [PATCH 184/264] Comments removed --- elastalert/zabbix.py | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/elastalert/zabbix.py b/elastalert/zabbix.py index d830ee3b9..f15d2d796 100644 --- a/elastalert/zabbix.py +++ b/elastalert/zabbix.py @@ -61,34 +61,12 @@ def alert(self, matches): # Matches is a list of match dictionaries. # It contains more than one match when the alert has # the aggregation option set - zm = [] -# zm = [ZabbixMetric(self.zbx_host, self.zbx_key, matches_len)] -# ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) for match in matches: - ts_epoch = int(datetime.strptime(match['@timestamp'], "%Y-%m-%dT%H:%M:%S.%fZ").strftime('%s')) zm.append(ZabbixMetric(host=self.zbx_host, key=self.zbx_key, value=1, clock=ts_epoch)) -# -# if match['operation']: -# key = '{0}[{1}]'.format(match['operation'], self.zbx_key) -# else: -# continue -# print(key) - -# zm = ZabbixMetric(self.zbx_host, key, 1) -# ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) - - # Config options can be accessed with self.rule -# with open(self.rule['output_file_path'], "a") as output_file: - -# # basic_match_string will transform the match into the default -# # human readable string format -# match_string = str(BasicMatchString(self.rule, match)) - -# output_file.write(match_string) ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) From 0db4e71234e5bd21e70958071c5f3bd5da647e60 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Tue, 21 May 2019 13:30:11 +0200 Subject: [PATCH 185/264] Doc added --- docs/source/ruletypes.rst | 12 ++++++++++++ elastalert/schema.yaml | 6 ++++++ 2 files changed, 18 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 113b2e50e..c8e732831 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2118,3 +2118,15 @@ Example usage:: - domain: "{match[field]}" - ip: "{match[ip_field]}" + +Zabbix +~~~~~~~~~~~ + +Zabbix will send notification to a Zabbix server. The item in the host specified receive a 1 value for each hit. For example, if the elastic query produce 3 hits in the last execution of elastalert, three '1' (integer) values will be send from elastalert to Zabbix Server. If the query have 0 hits, any value will be sent. + +Required: + +``zbx_sender_host``: The address where zabbix server is running. +``zbx_sender_port``: The port where zabbix server is listenning. +``zbx_host``: This field setup the host in zabbix that receives the value sent by Elastalert. +``zbx_item``: This field setup the item in the host that receives the value sent by Elastalert. \ No newline at end of file diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index e5caacc6e..06f9dfa60 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -342,3 +342,9 @@ properties: ### LineNotify linenotify_access_token: {type: string} + + ### Zabbix + zbx_sender_host: {type: string} + zbx_sender_port: {type: integer} + zbx_host: {type: string} + zbx_item: {type: string} From d8d2e15d9a8b3d964cc2cbce06c1146c20718c34 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Tue, 21 May 2019 13:38:12 +0200 Subject: [PATCH 186/264] E303 too many blank lines --- elastalert/zabbix.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/elastalert/zabbix.py b/elastalert/zabbix.py index f15d2d796..e3f13aa03 100644 --- a/elastalert/zabbix.py +++ b/elastalert/zabbix.py @@ -62,8 +62,6 @@ def alert(self, matches): # It contains more than one match when the alert has # the aggregation option set zm = [] - - for match in matches: ts_epoch = int(datetime.strptime(match['@timestamp'], "%Y-%m-%dT%H:%M:%S.%fZ").strftime('%s')) zm.append(ZabbixMetric(host=self.zbx_host, key=self.zbx_key, value=1, clock=ts_epoch)) From ed1ee6f4f2a85eef78cd2f6a6a117a6427102dae Mon Sep 17 00:00:00 2001 From: Florian GAULTIER Date: Mon, 3 Jun 2019 15:42:11 +0200 Subject: [PATCH 187/264] cleart scroll if not used anymore --- elastalert/elastalert.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index e08b89503..151cf615d 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -351,6 +351,7 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): ignore_unavailable=True, **extra_args ) + rule['scroll_id'] = res['_scroll_id'] if self.current_es.is_atleastseven(): self.total_hits = int(res['hits']['total']['value']) @@ -386,7 +387,6 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): ) if self.total_hits > rule.get('max_query_size', self.max_query_size): elastalert_logger.info("%s (scrolling..)" % status_log) - rule['scroll_id'] = res['_scroll_id'] else: elastalert_logger.info(status_log) @@ -624,7 +624,8 @@ def run_query(self, rule, start=None, end=None, scroll=False): pass if 'scroll_id' in rule: - rule.pop('scroll_id') + scroll_id = rule.pop('scroll_id') + self.current_es.clear_scroll(scroll_id=scroll_id) return True From 1b5cb2beb1dee090f4baf5eaedb2022b2aa03783 Mon Sep 17 00:00:00 2001 From: Florian GAULTIER Date: Mon, 3 Jun 2019 16:09:43 +0200 Subject: [PATCH 188/264] Verify _scroll_id exists to pass test --- elastalert/elastalert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 151cf615d..73a95824e 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -351,7 +351,8 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): ignore_unavailable=True, **extra_args ) - rule['scroll_id'] = res['_scroll_id'] + if '_scroll_id' in res: + rule['scroll_id'] = res['_scroll_id'] if self.current_es.is_atleastseven(): self.total_hits = int(res['hits']['total']['value']) From 74da1c7e328d65eaa14f79091ccd52f276189e7d Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 7 Jun 2019 15:40:44 -0700 Subject: [PATCH 189/264] Python3 supprt --- elastalert/__init__.py | 11 +- elastalert/alerts.py | 134 +++++++------ elastalert/config.py | 23 +-- elastalert/create_index.py | 25 ++- elastalert/elastalert.py | 93 ++++----- elastalert/enhancements.py | 2 +- elastalert/kibana.py | 341 +++++++++++++++++---------------- elastalert/loaders.py | 55 +++--- elastalert/opsgenie.py | 22 ++- elastalert/rule_from_kibana.py | 11 +- elastalert/ruletypes.py | 65 +++---- elastalert/test_rule.py | 9 +- elastalert/util.py | 10 +- setup.py | 2 +- tests/alerts_test.py | 114 +++++------ tests/base_test.py | 25 +-- tests/conftest.py | 2 +- tests/create_index_test.py | 12 +- tests/elasticsearch_test.py | 6 +- tests/loaders_test.py | 19 +- tox.ini | 2 +- 21 files changed, 483 insertions(+), 500 deletions(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 90b14126e..05d9ef7ff 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- import copy import logging -from elasticsearch import Elasticsearch, RequestsHttpConnection -from elasticsearch.client import query_params, _make_path + +from elasticsearch import Elasticsearch +from elasticsearch import RequestsHttpConnection +from elasticsearch.client import _make_path +from elasticsearch.client import query_params class ElasticSearchClient(Elasticsearch): @@ -59,14 +62,14 @@ def is_atleastsixtwo(self): """ Returns True when the Elasticsearch server version >= 6.2 """ - major, minor = map(int, self.es_version.split(".")[:2]) + major, minor = list(map(int, self.es_version.split(".")[:2])) return major > 6 or (major == 6 and minor >= 2) def is_atleastsixsix(self): """ Returns True when the Elasticsearch server version >= 6.6 """ - major, minor = map(int, self.es_version.split(".")[:2]) + major, minor = list(map(int, self.es_version.split(".")[:2])) return major > 6 or (major == 6 and minor >= 6) def is_atleastseven(self): diff --git a/elastalert/alerts.py b/elastalert/alerts.py index c02dae9b1..d3ee892d4 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -12,7 +12,7 @@ import warnings from email.mime.text import MIMEText from email.utils import formatdate -from HTMLParser import HTMLParser +from html.parser import HTMLParser from smtplib import SMTP from smtplib import SMTP_SSL from smtplib import SMTPAuthenticationError @@ -35,13 +35,14 @@ from thehive4py.models import CustomFieldHelper from twilio.base.exceptions import TwilioRestException from twilio.rest import Client as TwilioClient -from util import EAException -from util import elastalert_logger -from util import lookup_es_key -from util import pretty_ts -from util import resolve_string -from util import ts_now -from util import ts_to_dt + +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key +from .util import pretty_ts +from .util import resolve_string +from .util import ts_now +from .util import ts_to_dt class DateTimeEncoder(json.JSONEncoder): @@ -65,7 +66,7 @@ def _ensure_new_line(self): def _add_custom_alert_text(self): missing = self.rule.get('alert_missing_value', '') - alert_text = unicode(self.rule.get('alert_text', '')) + alert_text = str(self.rule.get('alert_text', '')) if 'alert_text_args' in self.rule: alert_text_args = self.rule.get('alert_text_args') alert_text_values = [lookup_es_key(self.match, arg) for arg in alert_text_args] @@ -83,7 +84,7 @@ def _add_custom_alert_text(self): alert_text = alert_text.format(*alert_text_values) elif 'alert_text_kw' in self.rule: kw = {} - for name, kw_name in self.rule.get('alert_text_kw').items(): + for name, kw_name in list(self.rule.get('alert_text_kw').items()): val = lookup_es_key(self.match, name) # Support referencing other top-level rule properties @@ -101,10 +102,10 @@ def _add_rule_text(self): self.text += self.rule['type'].get_match_str(self.match) def _add_top_counts(self): - for key, counts in self.match.items(): + for key, counts in list(self.match.items()): if key.startswith('top_events_'): self.text += '%s:\n' % (key[11:]) - top_events = counts.items() + top_events = list(counts.items()) if not top_events: self.text += 'No events found.\n' @@ -116,12 +117,12 @@ def _add_top_counts(self): self.text += '\n' def _add_match_items(self): - match_items = self.match.items() + match_items = list(self.match.items()) match_items.sort(key=lambda x: x[0]) for key, value in match_items: if key.startswith('top_events_'): continue - value_str = unicode(value) + value_str = str(value) value_str.replace('\\n', '\n') if type(value) in [list, dict]: try: @@ -157,9 +158,9 @@ def __str__(self): class JiraFormattedMatchString(BasicMatchString): def _add_match_items(self): - match_items = dict([(x, y) for x, y in self.match.items() if not x.startswith('top_events_')]) + match_items = dict([(x, y) for x, y in list(self.match.items()) if not x.startswith('top_events_')]) json_blob = self._pretty_print_as_json(match_items) - preformatted_text = u'{{code}}{0}{{code}}'.format(json_blob) + preformatted_text = '{{code}}{0}{{code}}'.format(json_blob) self.text += preformatted_text @@ -188,14 +189,14 @@ def resolve_rule_references(self, root): root[i] = self.resolve_rule_reference(item) elif type(root) == dict: # Make a copy since we may be modifying the contents of the structure we're walking - for key, value in root.copy().iteritems(): + for key, value in root.copy().items(): if type(value) == dict or type(value) == list: self.resolve_rule_references(root[key]) else: root[key] = self.resolve_rule_reference(value) def resolve_rule_reference(self, value): - strValue = unicode(value) + strValue = str(value) if strValue.startswith('$') and strValue.endswith('$') and strValue[1:-1] in self.rule: if type(value) == int: return int(self.rule[strValue[1:-1]]) @@ -227,7 +228,7 @@ def create_title(self, matches): return self.create_default_title(matches) def create_custom_title(self, matches): - alert_subject = unicode(self.rule['alert_subject']) + alert_subject = str(self.rule['alert_subject']) alert_subject_max_len = int(self.rule.get('alert_subject_max_len', 2048)) if 'alert_subject_args' in self.rule: @@ -256,7 +257,7 @@ def create_alert_body(self, matches): body = self.get_aggregation_summary_text(matches) if self.rule.get('alert_text_type') != 'aggregation_summary_only': for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' @@ -286,16 +287,16 @@ def get_aggregation_summary_text(self, matches): # Maintain an aggregate count for each unique key encountered in the aggregation period for match in matches: - key_tuple = tuple([unicode(lookup_es_key(match, key)) for key in summary_table_fields]) + key_tuple = tuple([str(lookup_es_key(match, key)) for key in summary_table_fields]) if key_tuple not in match_aggregation: match_aggregation[key_tuple] = 1 else: match_aggregation[key_tuple] = match_aggregation[key_tuple] + 1 - for keys, count in match_aggregation.iteritems(): + for keys, count in match_aggregation.items(): text_table.add_row([key for key in keys] + [count]) text += text_table.draw() + '\n\n' text += self.rule.get('summary_prefix', '') - return unicode(text) + return str(text) def create_default_title(self, matches): return self.rule['name'] @@ -351,13 +352,13 @@ def alert(self, matches): ) fullmessage['match'] = lookup_es_key( match, self.rule['timestamp_field']) - elastalert_logger.info(unicode(BasicMatchString(self.rule, match))) + elastalert_logger.info(str(BasicMatchString(self.rule, match))) fullmessage['alerts'] = alerts fullmessage['rule'] = self.rule['name'] fullmessage['rule_file'] = self.rule['rule_file'] - fullmessage['matching'] = unicode(BasicMatchString(self.rule, match)) + fullmessage['matching'] = str(BasicMatchString(self.rule, match)) fullmessage['alertDate'] = datetime.datetime.now( ).strftime("%Y-%m-%d %H:%M:%S") fullmessage['body'] = self.create_alert_body(matches) @@ -396,7 +397,7 @@ def alert(self, matches): 'Alert for %s, %s at %s:' % (self.rule['name'], match[qk], lookup_es_key(match, self.rule['timestamp_field']))) else: elastalert_logger.info('Alert for %s at %s:' % (self.rule['name'], lookup_es_key(match, self.rule['timestamp_field']))) - elastalert_logger.info(unicode(BasicMatchString(self.rule, match))) + elastalert_logger.info(str(BasicMatchString(self.rule, match))) def get_info(self): return {'type': 'debug'} @@ -418,15 +419,15 @@ def __init__(self, *args): self.smtp_key_file = self.rule.get('smtp_key_file') self.smtp_cert_file = self.rule.get('smtp_cert_file') # Convert email to a list if it isn't already - if isinstance(self.rule['email'], basestring): + if isinstance(self.rule['email'], str): self.rule['email'] = [self.rule['email']] # If there is a cc then also convert it a list if it isn't cc = self.rule.get('cc') - if cc and isinstance(cc, basestring): + if cc and isinstance(cc, str): self.rule['cc'] = [self.rule['cc']] # If there is a bcc then also convert it to a list if it isn't bcc = self.rule.get('bcc') - if bcc and isinstance(bcc, basestring): + if bcc and isinstance(bcc, str): self.rule['bcc'] = [self.rule['bcc']] add_suffix = self.rule.get('email_add_domain') if add_suffix and not add_suffix.startswith('@'): @@ -443,7 +444,7 @@ def alert(self, matches): to_addr = self.rule['email'] if 'email_from_field' in self.rule: recipient = lookup_es_key(matches[0], self.rule['email_from_field']) - if isinstance(recipient, basestring): + if isinstance(recipient, str): if '@' in recipient: to_addr = [recipient] elif 'email_add_domain' in self.rule: @@ -453,9 +454,9 @@ def alert(self, matches): if 'email_add_domain' in self.rule: to_addr = [name + self.rule['email_add_domain'] for name in to_addr] if self.rule.get('email_format') == 'html': - email_msg = MIMEText(body.encode('UTF-8'), 'html', _charset='UTF-8') + email_msg = MIMEText(body, 'html', _charset='UTF-8') else: - email_msg = MIMEText(body.encode('UTF-8'), _charset='UTF-8') + email_msg = MIMEText(body, _charset='UTF-8') email_msg['Subject'] = self.create_title(matches) email_msg['To'] = ', '.join(to_addr) email_msg['From'] = self.from_addr @@ -600,7 +601,7 @@ def __init__(self, rule): self.get_arbitrary_fields() except JIRAError as e: # JIRAError may contain HTML, pass along only first 1024 chars - raise EAException("Error connecting to JIRA: %s" % (str(e)[:1024])), None, sys.exc_info()[2] + raise EAException("Error connecting to JIRA: %s" % (str(e)[:1024])).with_traceback(sys.exc_info()[2]) self.set_priority() @@ -609,7 +610,7 @@ def set_priority(self): if self.priority is not None and self.client is not None: self.jira_args['priority'] = {'id': self.priority_ids[self.priority]} except KeyError: - logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, self.priority_ids.keys())) + logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, list(self.priority_ids.keys()))) def reset_jira_args(self): self.jira_args = {'project': {'key': self.project}, @@ -703,7 +704,7 @@ def get_arbitrary_fields(self): # Clear jira_args self.reset_jira_args() - for jira_field, value in self.rule.iteritems(): + for jira_field, value in self.rule.items(): # If we find a field that is not covered by the set that we are aware of, it means it is either: # 1. A built-in supported field in JIRA that we don't have on our radar # 2. A custom field that a JIRA admin has configured @@ -759,7 +760,7 @@ def find_existing_ticket(self, matches): return issues[0] def comment_on_ticket(self, ticket, match): - text = unicode(JiraFormattedMatchString(self.rule, match)) + text = str(JiraFormattedMatchString(self.rule, match)) timestamp = pretty_ts(lookup_es_key(match, self.rule['timestamp_field'])) comment = "This alert was triggered again at %s\n%s" % (timestamp, text) self.client.add_comment(ticket, comment) @@ -834,7 +835,7 @@ def alert(self, matches): "Exception encountered when trying to add '{0}' as a watcher. Does the user exist?\n{1}" .format( watcher, ex - )), None, sys.exc_info()[2] + )).with_traceback(sys.exc_info()[2]) except JIRAError as e: raise EAException("Error creating JIRA ticket using jira_args (%s): %s" % (self.jira_args, e)) @@ -849,7 +850,7 @@ def create_alert_body(self, matches): body += self.get_aggregation_summary_text(matches) if self.rule.get('alert_text_type') != 'aggregation_summary_only': for match in matches: - body += unicode(JiraFormattedMatchString(self.rule, match)) + body += str(JiraFormattedMatchString(self.rule, match)) if len(matches) > 1: body += '\n----------------------------------------\n' return body @@ -857,7 +858,7 @@ def create_alert_body(self, matches): def get_aggregation_summary_text(self, matches): text = super(JiraAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'{{noformat}}{0}{{noformat}}'.format(text) + text = '{{noformat}}{0}{{noformat}}'.format(text) return text def create_default_title(self, matches, for_search=False): @@ -893,7 +894,7 @@ def __init__(self, *args): self.last_command = [] self.shell = False - if isinstance(self.rule['command'], basestring): + if isinstance(self.rule['command'], str): self.shell = True if '%' in self.rule['command']: logging.warning('Warning! You could be vulnerable to shell injection!') @@ -1059,7 +1060,7 @@ class MsTeamsAlerter(Alerter): def __init__(self, rule): super(MsTeamsAlerter, self).__init__(rule) self.ms_teams_webhook_url = self.rule['ms_teams_webhook_url'] - if isinstance(self.ms_teams_webhook_url, basestring): + if isinstance(self.ms_teams_webhook_url, str): self.ms_teams_webhook_url = [self.ms_teams_webhook_url] self.ms_teams_proxy = self.rule.get('ms_teams_proxy', None) self.ms_teams_alert_summary = self.rule.get('ms_teams_alert_summary', 'ElastAlert Message') @@ -1067,7 +1068,6 @@ def __init__(self, rule): self.ms_teams_theme_color = self.rule.get('ms_teams_theme_color', '') def format_body(self, body): - body = body.encode('UTF-8') if self.ms_teams_alert_fixed_width: body = body.replace('`', "'") body = "```{0}```".format('```\n\n```'.join(x for x in body.split('\n'))).replace('\n``````', '') @@ -1111,12 +1111,12 @@ class SlackAlerter(Alerter): def __init__(self, rule): super(SlackAlerter, self).__init__(rule) self.slack_webhook_url = self.rule['slack_webhook_url'] - if isinstance(self.slack_webhook_url, basestring): + if isinstance(self.slack_webhook_url, str): self.slack_webhook_url = [self.slack_webhook_url] self.slack_proxy = self.rule.get('slack_proxy', None) self.slack_username_override = self.rule.get('slack_username_override', 'elastalert') self.slack_channel_override = self.rule.get('slack_channel_override', '') - if isinstance(self.slack_channel_override, basestring): + if isinstance(self.slack_channel_override, str): self.slack_channel_override = [self.slack_channel_override] self.slack_title_link = self.rule.get('slack_title_link', '') self.slack_title = self.rule.get('slack_title', '') @@ -1132,7 +1132,7 @@ def __init__(self, rule): def format_body(self, body): # https://api.slack.com/docs/formatting - return body.encode('UTF-8') + return body def get_aggregation_summary_text__maximum_width(self): width = super(SlackAlerter, self).get_aggregation_summary_text__maximum_width() @@ -1142,7 +1142,7 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = super(SlackAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'```\n{0}```\n'.format(text) + text = '```\n{0}```\n'.format(text) return text def populate_fields(self, matches): @@ -1226,7 +1226,7 @@ def __init__(self, rule): # HTTP config self.mattermost_webhook_url = self.rule['mattermost_webhook_url'] - if isinstance(self.mattermost_webhook_url, basestring): + if isinstance(self.mattermost_webhook_url, str): self.mattermost_webhook_url = [self.mattermost_webhook_url] self.mattermost_proxy = self.rule.get('mattermost_proxy', None) self.mattermost_ignore_ssl_errors = self.rule.get('mattermost_ignore_ssl_errors', False) @@ -1249,7 +1249,7 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = super(MattermostAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'```\n{0}```\n'.format(text) + text = '```\n{0}```\n'.format(text) return text def populate_fields(self, matches): @@ -1382,7 +1382,7 @@ def alert(self, matches): matches), 'summary': self.create_title(matches), 'custom_details': { - 'information': body.encode('UTF-8'), + 'information': body, }, }, } @@ -1397,7 +1397,7 @@ def alert(self, matches): 'incident_key': self.get_incident_key(matches), 'client': self.pagerduty_client_name, 'details': { - "information": body.encode('UTF-8'), + "information": body, }, } @@ -1513,7 +1513,7 @@ def alert(self, matches): if response != 200: raise EAException("Error posting to Exotel, response code is %s" % response) except RequestException: - raise EAException("Error posting to Exotel"), None, sys.exc_info()[2] + raise EAException("Error posting to Exotel").with_traceback(sys.exc_info()[2]) elastalert_logger.info("Trigger sent to Exotel") def get_info(self): @@ -1606,15 +1606,15 @@ def __init__(self, rule): self.telegram_proxy_password = self.rule.get('telegram_proxy_pass', None) def alert(self, matches): - body = u'⚠ *%s* ⚠ ```\n' % (self.create_title(matches)) + body = '⚠ *%s* ⚠ ```\n' % (self.create_title(matches)) for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' if len(body) > 4095: - body = body[0:4000] + u"\n⚠ *message was cropped according to telegram limits!* ⚠" - body += u' ```' + body = body[0:4000] + "\n⚠ *message was cropped according to telegram limits!* ⚠" + body += ' ```' headers = {'content-type': 'application/json'} # set https proxy, if it was provided @@ -1649,7 +1649,7 @@ class GoogleChatAlerter(Alerter): def __init__(self, rule): super(GoogleChatAlerter, self).__init__(rule) self.googlechat_webhook_url = self.rule['googlechat_webhook_url'] - if isinstance(self.googlechat_webhook_url, basestring): + if isinstance(self.googlechat_webhook_url, str): self.googlechat_webhook_url = [self.googlechat_webhook_url] self.googlechat_format = self.rule.get('googlechat_format', 'basic') self.googlechat_header_title = self.rule.get('googlechat_header_title', None) @@ -1689,7 +1689,7 @@ def create_card(self, matches): card = {"cards": [{ "sections": [{ "widgets": [ - {"textParagraph": {"text": self.create_alert_body(matches).encode('UTF-8')}} + {"textParagraph": {"text": self.create_alert_body(matches)}} ]} ]} ]} @@ -1707,7 +1707,6 @@ def create_card(self, matches): def create_basic(self, matches): body = self.create_alert_body(matches) - body = body.encode('UTF-8') return {'text': body} def alert(self, matches): @@ -1865,7 +1864,6 @@ def alert(self, matches): headers = {'content-type': 'application/json'} if self.api_key is not None: headers['Authorization'] = 'Key %s' % (self.rule['alerta_api_key']) - alerta_payload = self.get_json_payload(matches[0]) try: @@ -1924,8 +1922,8 @@ def get_json_payload(self, match): 'service': [resolve_string(a_service, match, self.missing_text) for a_service in self.service], 'tags': [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags], 'correlate': [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate], - 'attributes': dict(zip(self.attributes_keys, - [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values])), + 'attributes': dict(list(zip(self.attributes_keys, + [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values]))), 'rawData': self.create_alert_body([match]), } @@ -1942,7 +1940,7 @@ class HTTPPostAlerter(Alerter): def __init__(self, rule): super(HTTPPostAlerter, self).__init__(rule) post_url = self.rule.get('http_post_url') - if isinstance(post_url, basestring): + if isinstance(post_url, str): post_url = [post_url] self.post_url = post_url self.post_proxy = self.rule.get('http_post_proxy') @@ -1957,7 +1955,7 @@ def alert(self, matches): for match in matches: payload = match if self.post_all_values else {} payload.update(self.post_static_payload) - for post_key, es_key in self.post_payload.items(): + for post_key, es_key in list(self.post_payload.items()): payload[post_key] = lookup_es_key(match, es_key) headers = { "Content-Type": "application/json", @@ -2123,12 +2121,12 @@ def alert(self, matches): artifacts = [] for mapping in self.rule.get('hive_observable_data_mapping', []): - for observable_type, match_data_key in mapping.iteritems(): + for observable_type, match_data_key in mapping.items(): try: match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) data_keys = match_data_keys + rule_data_keys - context_keys = context['match'].keys() + context['rule'].keys() + context_keys = list(context['match'].keys()) + list(context['rule'].keys()) if all([True if k in context_keys else False for k in data_keys]): artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) except KeyError: @@ -2141,10 +2139,10 @@ def alert(self, matches): } alert_config.update(self.rule.get('hive_alert_config', {})) - for alert_config_field, alert_config_value in alert_config.iteritems(): + for alert_config_field, alert_config_value in alert_config.items(): if alert_config_field == 'customFields': custom_fields = CustomFieldHelper() - for cf_key, cf_value in alert_config_value.iteritems(): + for cf_key, cf_value in alert_config_value.items(): try: func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) except AttributeError: @@ -2152,7 +2150,7 @@ def alert(self, matches): value = cf_value['value'].format(**context) func(cf_key, value) alert_config[alert_config_field] = custom_fields.build() - elif isinstance(alert_config_value, basestring): + elif isinstance(alert_config_value, str): alert_config[alert_config_field] = alert_config_value.format(**context) elif isinstance(alert_config_value, (list, tuple)): formatted_list = [] diff --git a/elastalert/config.py b/elastalert/config.py index cc87a36d1..23dbf1d55 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -3,12 +3,13 @@ import logging import logging.config -import loaders from envparse import Env from staticconf.loader import yaml_loader -from util import EAException -from util import elastalert_logger -from util import get_module + +from . import loaders +from .util import EAException +from .util import elastalert_logger +from .util import get_module # Required global (config.yaml) configuration options @@ -46,21 +47,21 @@ def load_conf(args, defaults=None, overwrites=None): # init logging from config and set log levels according to command line options configure_logging(args, conf) - for env_var, conf_var in env_settings.items(): + for env_var, conf_var in list(env_settings.items()): val = env(env_var, None) if val is not None: conf[conf_var] = val - for key, value in (defaults.iteritems() if defaults is not None else []): + for key, value in (iter(defaults.items()) if defaults is not None else []): if key not in conf: conf[key] = value - for key, value in (overwrites.iteritems() if overwrites is not None else []): + for key, value in (iter(overwrites.items()) if overwrites is not None else []): conf[key] = value # Make sure we have all required globals - if required_globals - frozenset(conf.keys()): - raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) + if required_globals - frozenset(list(conf.keys())): + raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(list(conf.keys()))))) conf.setdefault('writeback_alias', 'elastalert_alerts') conf.setdefault('max_query_size', 10000) @@ -90,9 +91,9 @@ def load_conf(args, defaults=None, overwrites=None): conf['rules_loader'] = rules_loader # Make sure we have all the required globals for the loader # Make sure we have all required globals - if rules_loader.required_globals - frozenset(conf.keys()): + if rules_loader.required_globals - frozenset(list(conf.keys())): raise EAException( - '%s must contain %s' % (filename, ', '.join(rules_loader.required_globals - frozenset(conf.keys())))) + '%s must contain %s' % (filename, ', '.join(rules_loader.required_globals - frozenset(list(conf.keys()))))) return conf diff --git a/elastalert/create_index.py b/elastalert/create_index.py index 009310258..39e3cbb60 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -1,7 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function - import argparse import getpass import json @@ -10,13 +8,14 @@ import elasticsearch.helpers import yaml -from auth import Auth from elasticsearch import RequestsHttpConnection from elasticsearch.client import Elasticsearch from elasticsearch.client import IndicesClient from elasticsearch.exceptions import NotFoundError from envparse import Env +from .auth import Auth + env = Env(ES_USE_SSL=bool) @@ -139,7 +138,7 @@ def is_atleastsix(es_version): def is_atleastsixtwo(es_version): - major, minor = map(int, es_version.split(".")[:2]) + major, minor = list(map(int, es_version.split(".")[:2])) return major > 6 or (major == 6 and minor >= 2) @@ -213,32 +212,32 @@ def main(): username = args.username if args.username else None password = args.password if args.password else None aws_region = args.aws_region - host = args.host if args.host else raw_input('Enter Elasticsearch host: ') - port = args.port if args.port else int(raw_input('Enter Elasticsearch port: ')) + host = args.host if args.host else input('Enter Elasticsearch host: ') + port = args.port if args.port else int(input('Enter Elasticsearch port: ')) use_ssl = (args.ssl if args.ssl is not None - else raw_input('Use SSL? t/f: ').lower() in ('t', 'true')) + else input('Use SSL? t/f: ').lower() in ('t', 'true')) if use_ssl: verify_certs = (args.verify_certs if args.verify_certs is not None - else raw_input('Verify TLS certificates? t/f: ').lower() not in ('f', 'false')) + else input('Verify TLS certificates? t/f: ').lower() not in ('f', 'false')) else: verify_certs = True if args.no_auth is None and username is None: - username = raw_input('Enter optional basic-auth username (or leave blank): ') + username = input('Enter optional basic-auth username (or leave blank): ') password = getpass.getpass('Enter optional basic-auth password (or leave blank): ') url_prefix = (args.url_prefix if args.url_prefix is not None - else raw_input('Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): ')) + else input('Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): ')) send_get_body_as = args.send_get_body_as ca_certs = None client_cert = None client_key = None - index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ') + index = args.index if args.index is not None else input('New index name? (Default elastalert_status) ') if not index: index = 'elastalert_status' - alias = args.alias if args.alias is not None else raw_input('New alias name? (Default elastalert_alerts) ') + alias = args.alias if args.alias is not None else input('New alias name? (Default elastalert_alerts) ') if not alias: alias = 'elastalert_alias' old_index = (args.old_index if args.old_index is not None - else raw_input('Name of existing index to copy? (Default None) ')) + else input('Name of existing index to copy? (Default None) ')) timeout = args.timeout diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 6da26cabd..98e9dc8f0 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -18,37 +18,38 @@ from socket import error import dateutil.tz -import kibana import pytz -from alerts import DebugAlerter from apscheduler.schedulers.background import BackgroundScheduler -from config import load_conf from croniter import croniter from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException from elasticsearch.exceptions import TransportError -from enhancements import DropMatchException -from ruletypes import FlatlineRule -from util import add_raw_postfix -from util import cronite_datetime_to_timestamp -from util import dt_to_ts -from util import dt_to_unix -from util import EAException -from util import elastalert_logger -from util import elasticsearch_client -from util import format_index -from util import lookup_es_key -from util import parse_deadline -from util import parse_duration -from util import pretty_ts -from util import replace_dots_in_field_names -from util import seconds -from util import set_es_key -from util import total_seconds -from util import ts_add -from util import ts_now -from util import ts_to_dt -from util import unix_to_dt + +from . import kibana +from .alerts import DebugAlerter +from .config import load_conf +from .enhancements import DropMatchException +from .ruletypes import FlatlineRule +from .util import add_raw_postfix +from .util import cronite_datetime_to_timestamp +from .util import dt_to_ts +from .util import dt_to_unix +from .util import EAException +from .util import elastalert_logger +from .util import elasticsearch_client +from .util import format_index +from .util import lookup_es_key +from .util import parse_deadline +from .util import parse_duration +from .util import pretty_ts +from .util import replace_dots_in_field_names +from .util import seconds +from .util import set_es_key +from .util import total_seconds +from .util import ts_add +from .util import ts_now +from .util import ts_to_dt +from .util import unix_to_dt class ElastAlerter(object): @@ -134,7 +135,7 @@ def __init__(self, args): self.rules_loader = self.conf['rules_loader'] self.rules = self.rules_loader.load(self.conf, self.args) - print len(self.rules), 'rules loaded' + print(len(self.rules), 'rules loaded') self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] @@ -166,7 +167,7 @@ def __init__(self, args): for rule in self.rules: if not self.init_rule(rule): remove.append(rule) - map(self.rules.remove, remove) + list(map(self.rules.remove, remove)) if self.args.silence: self.silence() @@ -306,7 +307,7 @@ def process_hits(rule, hits): for hit in hits: # Merge fields and _source hit.setdefault('_source', {}) - for key, value in hit.get('fields', {}).items(): + for key, value in list(hit.get('fields', {}).items()): # Fields are returned as lists, assume any with length 1 are not arrays in _source # Except sometimes they aren't lists. This is dependent on ES version hit['_source'].setdefault(key, value[0] if type(value) is list and len(value) == 1 else value) @@ -328,11 +329,11 @@ def process_hits(rule, hits): if rule.get('compound_query_key'): values = [lookup_es_key(hit['_source'], key) for key in rule['compound_query_key']] - hit['_source'][rule['query_key']] = ', '.join([unicode(value) for value in values]) + hit['_source'][rule['query_key']] = ', '.join([str(value) for value in values]) if rule.get('compound_aggregation_key'): values = [lookup_es_key(hit['_source'], key) for key in rule['compound_aggregation_key']] - hit['_source'][rule['aggregation_key']] = ', '.join([unicode(value) for value in values]) + hit['_source'][rule['aggregation_key']] = ', '.join([str(value) for value in values]) processed_hits.append(hit['_source']) @@ -595,10 +596,10 @@ def remove_old_events(self, rule): buffer_time = rule.get('buffer_time', self.buffer_time) if rule.get('query_delay'): buffer_time += rule['query_delay'] - for _id, timestamp in rule['processed_hits'].iteritems(): + for _id, timestamp in rule['processed_hits'].items(): if now - timestamp > buffer_time: remove.append(_id) - map(rule['processed_hits'].pop, remove) + list(map(rule['processed_hits'].pop, remove)) def run_query(self, rule, start=None, end=None, scroll=False): """ Query for the rule and pass all of the results to the RuleType instance. @@ -772,7 +773,7 @@ def get_query_key_value(self, rule, match): # get the value for the match's query_key (or none) to form the key used for the silence_cache. # Flatline ruletype sets "key" instead of the actual query_key if isinstance(rule['type'], FlatlineRule) and 'key' in match: - return unicode(match['key']) + return str(match['key']) return self.get_named_key_value(rule, match, 'query_key') def get_aggregation_key_value(self, rule, match): @@ -787,7 +788,7 @@ def get_named_key_value(self, rule, match, key_name): if key_value is not None: # Only do the unicode conversion if we actually found something) # otherwise we might transform None --> 'None' - key_value = unicode(key_value) + key_value = str(key_value) except KeyError: # Some matches may not have the specified key # use a special token for these @@ -1051,7 +1052,7 @@ def load_rule_changes(self): new_rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) # Check each current rule for changes - for rule_file, hash_value in self.rule_hashes.iteritems(): + for rule_file, hash_value in self.rule_hashes.items(): if rule_file not in new_rule_hashes: # Rule file was deleted elastalert_logger.info('Rule file %s not found, stopping rule execution' % (rule_file)) @@ -1227,7 +1228,7 @@ def handle_rule_execution(self, rule): if rule.get('limit_execution'): rule['next_starttime'] = None rule['next_min_starttime'] = None - exec_next = croniter(rule['limit_execution']).next() + exec_next = next(croniter(rule['limit_execution'])) endtime_epoch = dt_to_unix(endtime) # If the estimated next endtime (end + run_every) isn't at least a minute past the next exec time # That means that we need to pause execution after this run @@ -1383,7 +1384,7 @@ def get_dashboard(self, rule, db_name): # TODO use doc_type = _doc res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) except ElasticsearchException as e: - raise EAException("Error querying for dashboard: %s" % (e)), None, sys.exc_info()[2] + raise EAException("Error querying for dashboard: %s" % (e)).with_traceback(sys.exc_info()[2]) if res['hits']['hits']: return json.loads(res['hits']['hits'][0]['_source']['dashboard']) @@ -1485,7 +1486,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): try: enhancement.process(match) valid_matches.append(match) - except DropMatchException as e: + except DropMatchException: pass except EAException as e: self.handle_error("Error running match enhancement: %s" % (e), {'rule': rule['name']}) @@ -1560,7 +1561,7 @@ def writeback(self, doc_type, body, rule=None, match_body=None): else: writeback_body = body - for key in writeback_body.keys(): + for key in list(writeback_body.keys()): # Convert any datetime objects to timestamps if isinstance(writeback_body[key], datetime.datetime): writeback_body[key] = dt_to_ts(writeback_body[key]) @@ -1649,7 +1650,7 @@ def send_pending_alerts(self): self.alert([match_body], rule, alert_time=alert_time, retried=retried) if rule['current_aggregate_id']: - for qk, agg_id in rule['current_aggregate_id'].iteritems(): + for qk, agg_id in rule['current_aggregate_id'].items(): if agg_id == _id: rule['current_aggregate_id'].pop(qk) break @@ -1666,7 +1667,7 @@ def send_pending_alerts(self): # Send in memory aggregated alerts for rule in self.rules: if rule['agg_matches']: - for aggregation_key_value, aggregate_alert_time in rule['aggregate_alert_time'].iteritems(): + for aggregation_key_value, aggregate_alert_time in rule['aggregate_alert_time'].items(): if ts_now() > aggregate_alert_time: alertable_matches = [ agg_match @@ -1877,7 +1878,7 @@ def is_silenced(self, rule_name): if res['hits']['hits']: until_ts = res['hits']['hits'][0]['_source']['until'] exponent = res['hits']['hits'][0]['_source'].get('exponent', 0) - if rule_name not in self.silence_cache.keys(): + if rule_name not in list(self.silence_cache.keys()): self.silence_cache[rule_name] = (ts_to_dt(until_ts), exponent) else: self.silence_cache[rule_name] = (ts_to_dt(until_ts), self.silence_cache[rule_name][1]) @@ -1925,13 +1926,13 @@ def send_notification_email(self, text='', exception=None, rule=None, subject=No tb = traceback.format_exc() email_body += tb - if isinstance(self.notify_email, basestring): + if isinstance(self.notify_email, str): self.notify_email = [self.notify_email] email = MIMEText(email_body) email['Subject'] = subject if subject else 'ElastAlert notification' recipients = self.notify_email if rule and rule.get('notify_email'): - if isinstance(rule['notify_email'], basestring): + if isinstance(rule['notify_email'], str): rule['notify_email'] = [rule['notify_email']] recipients = recipients + rule['notify_email'] recipients = list(set(recipients)) @@ -1958,14 +1959,14 @@ def get_top_counts(self, rule, starttime, endtime, keys, number=None, qk=None): if hits_terms is None: top_events_count = {} else: - buckets = hits_terms.values()[0] + buckets = list(hits_terms.values())[0] # get_hits_terms adds to num_hits, but we don't want to count these self.thread_data.num_hits -= len(buckets) terms = {} for bucket in buckets: terms[bucket['key']] = bucket['doc_count'] - counts = terms.items() + counts = list(terms.items()) counts.sort(key=lambda x: x[1], reverse=True) top_events_count = dict(counts[:number]) diff --git a/elastalert/enhancements.py b/elastalert/enhancements.py index 2744e35c8..6cc1cdd57 100644 --- a/elastalert/enhancements.py +++ b/elastalert/enhancements.py @@ -1,5 +1,5 @@ # -*- coding: utf-8 -*- -from util import pretty_ts +from .util import pretty_ts class BaseEnhancement(object): diff --git a/elastalert/kibana.py b/elastalert/kibana.py index 2cd557bff..de690494e 100644 --- a/elastalert/kibana.py +++ b/elastalert/kibana.py @@ -1,173 +1,176 @@ # -*- coding: utf-8 -*- +# flake8: noqa import os.path -import urllib +import urllib.error +import urllib.parse +import urllib.request -from util import EAException +from .util import EAException dashboard_temp = {'editable': True, - u'failover': False, - u'index': {u'default': u'NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED', - u'interval': u'none', - u'pattern': u'', - u'warm_fields': True}, - u'loader': {u'hide': False, - u'load_elasticsearch': True, - u'load_elasticsearch_size': 20, - u'load_gist': True, - u'load_local': True, - u'save_default': True, - u'save_elasticsearch': True, - u'save_gist': False, - u'save_local': True, - u'save_temp': True, - u'save_temp_ttl': u'30d', - u'save_temp_ttl_enable': True}, - u'nav': [{u'collapse': False, - u'enable': True, - u'filter_id': 0, - u'notice': False, - u'now': False, - u'refresh_intervals': [u'5s', - u'10s', - u'30s', - u'1m', - u'5m', - u'15m', - u'30m', - u'1h', - u'2h', - u'1d'], - u'status': u'Stable', - u'time_options': [u'5m', - u'15m', - u'1h', - u'6h', - u'12h', - u'24h', - u'2d', - u'7d', - u'30d'], - u'timefield': u'@timestamp', - u'type': u'timepicker'}], - u'panel_hints': True, - u'pulldowns': [{u'collapse': False, - u'enable': True, - u'notice': True, - u'type': u'filtering'}], - u'refresh': False, - u'rows': [{u'collapsable': True, - u'collapse': False, - u'editable': True, - u'height': u'350px', - u'notice': False, - u'panels': [{u'annotate': {u'enable': False, - u'field': u'_type', - u'query': u'*', - u'size': 20, - u'sort': [u'_score', u'desc']}, - u'auto_int': True, - u'bars': True, - u'derivative': False, - u'editable': True, - u'fill': 3, - u'grid': {u'max': None, u'min': 0}, - u'group': [u'default'], - u'interactive': True, - u'interval': u'1m', - u'intervals': [u'auto', - u'1s', - u'1m', - u'5m', - u'10m', - u'30m', - u'1h', - u'3h', - u'12h', - u'1d', - u'1w', - u'1M', - u'1y'], - u'legend': True, - u'legend_counts': True, - u'lines': False, - u'linewidth': 3, - u'mode': u'count', - u'options': True, - u'percentage': False, - u'pointradius': 5, - u'points': False, - u'queries': {u'ids': [0], u'mode': u'all'}, - u'resolution': 100, - u'scale': 1, - u'show_query': True, - u'span': 12, - u'spyable': True, - u'stack': True, - u'time_field': u'@timestamp', - u'timezone': u'browser', - u'title': u'Events over time', - u'tooltip': {u'query_as_alias': True, - u'value_type': u'cumulative'}, - u'type': u'histogram', - u'value_field': None, - u'x-axis': True, - u'y-axis': True, - u'y_format': u'none', - u'zerofill': True, - u'zoomlinks': True}], - u'title': u'Graph'}, - {u'collapsable': True, - u'collapse': False, - u'editable': True, - u'height': u'350px', - u'notice': False, - u'panels': [{u'all_fields': False, - u'editable': True, - u'error': False, - u'field_list': True, - u'fields': [], - u'group': [u'default'], - u'header': True, - u'highlight': [], - u'localTime': True, - u'normTimes': True, - u'offset': 0, - u'overflow': u'min-height', - u'pages': 5, - u'paging': True, - u'queries': {u'ids': [0], u'mode': u'all'}, - u'size': 100, - u'sort': [u'@timestamp', u'desc'], - u'sortable': True, - u'span': 12, - u'spyable': True, - u'status': u'Stable', - u'style': {u'font-size': u'9pt'}, - u'timeField': u'@timestamp', - u'title': u'All events', - u'trimFactor': 300, - u'type': u'table'}], - u'title': u'Events'}], - u'services': {u'filter': {u'ids': [0], - u'list': {u'0': {u'active': True, - u'alias': u'', - u'field': u'@timestamp', - u'from': u'now-24h', - u'id': 0, - u'mandate': u'must', - u'to': u'now', - u'type': u'time'}}}, - u'query': {u'ids': [0], - u'list': {u'0': {u'alias': u'', - u'color': u'#7EB26D', - u'enable': True, - u'id': 0, - u'pin': False, - u'query': u'', - u'type': u'lucene'}}}}, - u'style': u'dark', - u'title': u'ElastAlert Alert Dashboard'} + 'failover': False, + 'index': {'default': 'NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED', + 'interval': 'none', + 'pattern': '', + 'warm_fields': True}, + 'loader': {'hide': False, + 'load_elasticsearch': True, + 'load_elasticsearch_size': 20, + 'load_gist': True, + 'load_local': True, + 'save_default': True, + 'save_elasticsearch': True, + 'save_gist': False, + 'save_local': True, + 'save_temp': True, + 'save_temp_ttl': '30d', + 'save_temp_ttl_enable': True}, + 'nav': [{'collapse': False, + 'enable': True, + 'filter_id': 0, + 'notice': False, + 'now': False, + 'refresh_intervals': ['5s', + '10s', + '30s', + '1m', + '5m', + '15m', + '30m', + '1h', + '2h', + '1d'], + 'status': 'Stable', + 'time_options': ['5m', + '15m', + '1h', + '6h', + '12h', + '24h', + '2d', + '7d', + '30d'], + 'timefield': '@timestamp', + 'type': 'timepicker'}], + 'panel_hints': True, + 'pulldowns': [{'collapse': False, + 'enable': True, + 'notice': True, + 'type': 'filtering'}], + 'refresh': False, + 'rows': [{'collapsable': True, + 'collapse': False, + 'editable': True, + 'height': '350px', + 'notice': False, + 'panels': [{'annotate': {'enable': False, + 'field': '_type', + 'query': '*', + 'size': 20, + 'sort': ['_score', 'desc']}, + 'auto_int': True, + 'bars': True, + 'derivative': False, + 'editable': True, + 'fill': 3, + 'grid': {'max': None, 'min': 0}, + 'group': ['default'], + 'interactive': True, + 'interval': '1m', + 'intervals': ['auto', + '1s', + '1m', + '5m', + '10m', + '30m', + '1h', + '3h', + '12h', + '1d', + '1w', + '1M', + '1y'], + 'legend': True, + 'legend_counts': True, + 'lines': False, + 'linewidth': 3, + 'mode': 'count', + 'options': True, + 'percentage': False, + 'pointradius': 5, + 'points': False, + 'queries': {'ids': [0], 'mode': 'all'}, + 'resolution': 100, + 'scale': 1, + 'show_query': True, + 'span': 12, + 'spyable': True, + 'stack': True, + 'time_field': '@timestamp', + 'timezone': 'browser', + 'title': 'Events over time', + 'tooltip': {'query_as_alias': True, + 'value_type': 'cumulative'}, + 'type': 'histogram', + 'value_field': None, + 'x-axis': True, + 'y-axis': True, + 'y_format': 'none', + 'zerofill': True, + 'zoomlinks': True}], + 'title': 'Graph'}, + {'collapsable': True, + 'collapse': False, + 'editable': True, + 'height': '350px', + 'notice': False, + 'panels': [{'all_fields': False, + 'editable': True, + 'error': False, + 'field_list': True, + 'fields': [], + 'group': ['default'], + 'header': True, + 'highlight': [], + 'localTime': True, + 'normTimes': True, + 'offset': 0, + 'overflow': 'min-height', + 'pages': 5, + 'paging': True, + 'queries': {'ids': [0], 'mode': 'all'}, + 'size': 100, + 'sort': ['@timestamp', 'desc'], + 'sortable': True, + 'span': 12, + 'spyable': True, + 'status': 'Stable', + 'style': {'font-size': '9pt'}, + 'timeField': '@timestamp', + 'title': 'All events', + 'trimFactor': 300, + 'type': 'table'}], + 'title': 'Events'}], + 'services': {'filter': {'ids': [0], + 'list': {'0': {'active': True, + 'alias': '', + 'field': '@timestamp', + 'from': 'now-24h', + 'id': 0, + 'mandate': 'must', + 'to': 'now', + 'type': 'time'}}}, + 'query': {'ids': [0], + 'list': {'0': {'alias': '', + 'color': '#7EB26D', + 'enable': True, + 'id': 0, + 'pin': False, + 'query': '', + 'type': 'lucene'}}}}, + 'style': 'dark', + 'title': 'ElastAlert Alert Dashboard'} kibana4_time_temp = "(refreshInterval:(display:Off,section:0,value:0),time:(from:'%s',mode:absolute,to:'%s'))" @@ -213,9 +216,9 @@ def add_filter(dashboard, es_filter): kibana_filter['query'] = es_filter['query_string']['query'] elif 'term' in es_filter: kibana_filter['type'] = 'field' - f_field, f_query = es_filter['term'].items()[0] + f_field, f_query = list(es_filter['term'].items())[0] # Wrap query in quotes, otherwise certain characters cause Kibana to throw errors - if isinstance(f_query, basestring): + if isinstance(f_query, str): f_query = '"%s"' % (f_query.replace('"', '\\"')) if isinstance(f_query, list): # Escape quotes @@ -228,7 +231,7 @@ def add_filter(dashboard, es_filter): kibana_filter['query'] = f_query elif 'range' in es_filter: kibana_filter['type'] = 'range' - f_field, f_range = es_filter['range'].items()[0] + f_field, f_range = list(es_filter['range'].items())[0] kibana_filter['field'] = f_field kibana_filter.update(f_range) else: @@ -250,7 +253,7 @@ def filters_from_dashboard(db): filters = db['services']['filter']['list'] config_filters = [] or_filters = [] - for filter in filters.values(): + for filter in list(filters.values()): filter_type = filter['type'] if filter_type == 'time': continue @@ -281,5 +284,5 @@ def filters_from_dashboard(db): def kibana4_dashboard_link(dashboard, starttime, endtime): dashboard = os.path.expandvars(dashboard) time_settings = kibana4_time_temp % (starttime, endtime) - time_settings = urllib.quote(time_settings) + time_settings = urllib.parse.quote(time_settings) return "%s?_g=%s" % (dashboard, time_settings) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index a1cadd342..db14a3a52 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -6,24 +6,25 @@ import os import sys -import alerts -import enhancements import jsonschema -import ruletypes import yaml import yaml.scanner -from opsgenie import OpsGenieAlerter from staticconf.loader import yaml_loader -from util import get_module -from util import dt_to_ts -from util import dt_to_ts_with_format -from util import dt_to_unix -from util import dt_to_unixms -from util import EAException -from util import ts_to_dt -from util import ts_to_dt_with_format -from util import unix_to_dt -from util import unixms_to_dt + +from . import alerts +from . import enhancements +from . import ruletypes +from .opsgenie import OpsGenieAlerter +from .util import dt_to_ts +from .util import dt_to_ts_with_format +from .util import dt_to_unix +from .util import dt_to_unixms +from .util import EAException +from .util import get_module +from .util import ts_to_dt +from .util import ts_to_dt_with_format +from .util import unix_to_dt +from .util import unixms_to_dt class RulesLoader(object): @@ -90,7 +91,7 @@ class RulesLoader(object): def __init__(self, conf): # schema for rule yaml self.rule_schema = jsonschema.Draft4Validator( - yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) + yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader)) self.base_config = copy.deepcopy(conf) @@ -260,7 +261,7 @@ def load_options(self, rule, conf, filename, args=None): raise EAException('Invalid time format used: %s' % e) # Set defaults, copy defaults from config.yaml - for key, val in self.base_config.items(): + for key, val in list(self.base_config.items()): rule.setdefault(key, val) rule.setdefault('name', os.path.splitext(filename)[0]) rule.setdefault('realert', datetime.timedelta(seconds=0)) @@ -317,8 +318,8 @@ def _dt_to_ts_with_format(dt): rule.setdefault('hipchat_ignore_ssl_errors', False) # Make sure we have required options - if self.required_locals - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(rule.keys())))) + if self.required_locals - frozenset(list(rule.keys())): + raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(list(rule.keys()))))) if 'include' in rule and type(rule['include']) != list: raise EAException('include option must be a list') @@ -361,7 +362,7 @@ def _dt_to_ts_with_format(dt): es_filter = es_filter['not'] if 'query' in es_filter: es_filter = es_filter['query'] - if es_filter.keys()[0] not in ('term', 'query_string', 'range'): + if list(es_filter.keys())[0] not in ('term', 'query_string', 'range'): raise EAException( 'generate_kibana_link is incompatible with filters other than term, query_string and range.' 'Consider creating a dashboard and using use_kibana_dashboard instead.') @@ -414,13 +415,13 @@ def load_modules(self, rule, args=None): # Make sure we have required alert and type options reqs = rule['type'].required_options - if reqs - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(rule.keys())))) + if reqs - frozenset(list(rule.keys())): + raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(list(rule.keys()))))) # Instantiate rule try: rule['type'] = rule['type'](rule, args) except (KeyError, EAException) as e: - raise EAException('Error initializing rule %s: %s' % (rule['name'], e)), None, sys.exc_info()[2] + raise EAException('Error initializing rule %s: %s' % (rule['name'], e)).with_traceback(sys.exc_info()[2]) # Instantiate alerts only if we're not in debug mode # In debug mode alerts are not actually sent so don't bother instantiating them if not args or not args.debug: @@ -430,10 +431,10 @@ def load_alerts(self, rule, alert_field): def normalize_config(alert): """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. This function normalizes them both to the latter format. """ - if isinstance(alert, basestring): + if isinstance(alert, str): return alert, rule elif isinstance(alert, dict): - name, config = iter(alert.items()).next() + name, config = next(iter(list(alert.items()))) config_copy = copy.copy(rule) config_copy.update(config) # warning, this (intentionally) mutates the rule dict return name, config_copy @@ -455,12 +456,12 @@ def create_alert(alert, alert_config): alert_field = [alert_field] alert_field = [normalize_config(x) for x in alert_field] - alert_field = sorted(alert_field, key=lambda (a, b): self.alerts_order.get(a, 1)) + alert_field = sorted(alert_field, key=lambda a_b: self.alerts_order.get(a_b[0], 1)) # Convert all alerts into Alerter objects alert_field = [create_alert(a, b) for a, b in alert_field] except (KeyError, EAException) as e: - raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)), None, sys.exc_info()[2] + raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)).with_traceback(sys.exc_info()[2]) return alert_field @@ -530,7 +531,7 @@ def get_import_rule(self, rule): def get_rule_file_hash(self, rule_file): rule_file_hash = '' if os.path.exists(rule_file): - with open(rule_file) as fh: + with open(rule_file, 'rb') as fh: rule_file_hash = hashlib.sha1(fh.read()).digest() for import_rule_file in self.import_rules.get(rule_file, []): rule_file_hash += self.get_rule_file_hash(import_rule_file) diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index 6a6e7fda7..8f58b0b26 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- import json import logging + import requests -from alerts import Alerter -from alerts import BasicMatchString -from util import EAException -from util import elastalert_logger -from util import lookup_es_key + +from .alerts import Alerter +from .alerts import BasicMatchString +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key class OpsGenieAlerter(Alerter): @@ -35,11 +37,11 @@ def __init__(self, *args): def _parse_responders(self, responders, responder_args, matches, default_responders): if responder_args: formated_responders = list() - responders_values = dict((k, lookup_es_key(matches[0], v)) for k, v in responder_args.iteritems()) - responders_values = dict((k, v) for k, v in responders_values.iteritems() if v) + responders_values = dict((k, lookup_es_key(matches[0], v)) for k, v in responder_args.items()) + responders_values = dict((k, v) for k, v in responders_values.items() if v) for responder in responders: - responder = unicode(responder) + responder = str(responder) try: formated_responders.append(responder.format(**responders_values)) except KeyError as error: @@ -60,7 +62,7 @@ def _fill_responders(self, responders, type_): def alert(self, matches): body = '' for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' @@ -131,7 +133,7 @@ def create_title(self, matches): return self.create_default_title(matches) def create_custom_title(self, matches): - opsgenie_subject = unicode(self.rule['opsgenie_subject']) + opsgenie_subject = str(self.rule['opsgenie_subject']) if self.opsgenie_subject_args: opsgenie_subject_values = [lookup_es_key(matches[0], arg) for arg in self.opsgenie_subject_args] diff --git a/elastalert/rule_from_kibana.py b/elastalert/rule_from_kibana.py index ef1392b28..4a0634954 100644 --- a/elastalert/rule_from_kibana.py +++ b/elastalert/rule_from_kibana.py @@ -1,8 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function - import json import yaml @@ -12,10 +9,10 @@ def main(): - es_host = raw_input("Elasticsearch host: ") - es_port = raw_input("Elasticsearch port: ") - db_name = raw_input("Dashboard name: ") - send_get_body_as = raw_input("Method for querying Elasticsearch[GET]: ") or 'GET' + es_host = input("Elasticsearch host: ") + es_port = input("Elasticsearch port: ") + db_name = input("Dashboard name: ") + send_get_body_as = input("Method for querying Elasticsearch[GET]: ") or 'GET' es = elasticsearch_client({'es_host': es_host, 'es_port': es_port, 'send_get_body_as': send_get_body_as}) diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index a8845ca92..2f1d2f82c 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -4,19 +4,20 @@ import sys from blist import sortedlist -from util import add_raw_postfix -from util import dt_to_ts -from util import EAException -from util import elastalert_logger -from util import elasticsearch_client -from util import format_index -from util import hashable -from util import lookup_es_key -from util import new_get_event_ts -from util import pretty_ts -from util import total_seconds -from util import ts_now -from util import ts_to_dt + +from .util import add_raw_postfix +from .util import dt_to_ts +from .util import EAException +from .util import elastalert_logger +from .util import elasticsearch_client +from .util import format_index +from .util import hashable +from .util import lookup_es_key +from .util import new_get_event_ts +from .util import pretty_ts +from .util import total_seconds +from .util import ts_now +from .util import ts_to_dt class RuleType(object): @@ -205,8 +206,8 @@ def add_match(self, match): if change: extra = {'old_value': change[0], 'new_value': change[1]} - elastalert_logger.debug("Description of the changed records " + str(dict(match.items() + extra.items()))) - super(ChangeRule, self).add_match(dict(match.items() + extra.items())) + elastalert_logger.debug("Description of the changed records " + str(dict(list(match.items()) + list(extra.items())))) + super(ChangeRule, self).add_match(dict(list(match.items()) + list(extra.items()))) class FrequencyRule(RuleType): @@ -224,14 +225,14 @@ def add_count_data(self, data): if len(data) > 1: raise EAException('add_count_data can only accept one count at a time') - (ts, count), = data.items() + (ts, count), = list(data.items()) event = ({self.ts_field: ts}, count) self.occurrences.setdefault('all', EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts)).append(event) self.check_for_match('all') def add_terms_data(self, terms): - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: event = ({self.ts_field: timestamp, self.rules['query_key']: bucket['key']}, bucket['doc_count']) @@ -274,10 +275,10 @@ def check_for_match(self, key, end=False): def garbage_collect(self, timestamp): """ Remove all occurrence data that is beyond the timeframe away """ stale_keys = [] - for key, window in self.occurrences.iteritems(): + for key, window in self.occurrences.items(): if timestamp - lookup_es_key(window.data[-1][0], self.ts_field) > self.rules['timeframe']: stale_keys.append(key) - map(self.occurrences.pop, stale_keys) + list(map(self.occurrences.pop, stale_keys)) def get_match_str(self, match): lt = self.rules.get('use_local_time') @@ -401,11 +402,11 @@ def add_count_data(self, data): """ Add count data to the rule. Data should be of the form {ts: count}. """ if len(data) > 1: raise EAException('add_count_data can only accept one count at a time') - for ts, count in data.iteritems(): + for ts, count in data.items(): self.handle_event({self.ts_field: ts}, count, 'all') def add_terms_data(self, terms): - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: count = bucket['doc_count'] event = {self.ts_field: timestamp, @@ -489,7 +490,7 @@ def add_match(self, match, qk): extra_info = {'spike_count': spike_count, 'reference_count': reference_count} - match = dict(match.items() + extra_info.items()) + match = dict(list(match.items()) + list(extra_info.items())) super(SpikeRule, self).add_match(match) @@ -535,7 +536,7 @@ def get_match_str(self, match): def garbage_collect(self, ts): # Windows are sized according to their newest event # This is a placeholder to accurately size windows in the absence of events - for qk in self.cur_windows.keys(): + for qk in list(self.cur_windows.keys()): # If we havn't seen this key in a long time, forget it if qk != 'all' and self.ref_windows[qk].count() == 0 and self.cur_windows[qk].count() == 0: self.cur_windows.pop(qk) @@ -608,7 +609,7 @@ def garbage_collect(self, ts): # We add an event with a count of zero to the EventWindow for each key. This will cause the EventWindow # to remove events that occurred more than one `timeframe` ago, and call onRemoved on them. default = ['all'] if 'query_key' not in self.rules else [] - for key in self.occurrences.keys() or default: + for key in list(self.occurrences.keys()) or default: self.occurrences.setdefault( key, EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts) @@ -650,7 +651,7 @@ def __init__(self, rule, args=None): self.get_all_terms(args) except Exception as e: # Refuse to start if we cannot get existing terms - raise EAException('Error searching for existing terms: %s' % (repr(e))), None, sys.exc_info()[2] + raise EAException('Error searching for existing terms: %s' % (repr(e))).with_traceback(sys.exc_info()[2]) def get_all_terms(self, args): """ Performs a terms aggregation for each field to get every existing term. """ @@ -731,7 +732,7 @@ def get_all_terms(self, args): time_filter[self.rules['timestamp_field']] = {'lt': self.rules['dt_to_ts'](tmp_end), 'gte': self.rules['dt_to_ts'](tmp_start)} - for key, values in self.seen_values.iteritems(): + for key, values in self.seen_values.items(): if not values: if type(key) == tuple: # If we don't have any results, it could either be because of the absence of any baseline data @@ -879,7 +880,7 @@ def add_data(self, data): def add_terms_data(self, terms): # With terms query, len(self.fields) is always 1 and the 0'th entry is always a string field = self.fields[0] - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: if bucket['doc_count']: if bucket['key'] not in self.seen_values[field]: @@ -941,8 +942,8 @@ def check_for_match(self, key, event, gc=True): def garbage_collect(self, timestamp): """ Remove all occurrence data that is beyond the timeframe away """ - for qk, terms in self.cardinality_cache.items(): - for term, last_occurence in terms.items(): + for qk, terms in list(self.cardinality_cache.items()): + for term, last_occurence in list(terms.items()): if timestamp - last_occurence > self.rules['timeframe']: self.cardinality_cache[qk].pop(term) @@ -997,7 +998,7 @@ def generate_aggregation_query(self): raise NotImplementedError() def add_aggregation_data(self, payload): - for timestamp, payload_data in payload.iteritems(): + for timestamp, payload_data in payload.items(): if 'interval_aggs' in payload_data: self.unwrap_interval_buckets(timestamp, None, payload_data['interval_aggs']['buckets']) elif 'bucket_aggs' in payload_data: @@ -1087,7 +1088,7 @@ def check_matches_recursive(self, timestamp, query_key, aggregation_data, compou # add compound key to payload to allow alerts to trigger for every unique occurence compound_value = [match_data[key] for key in self.rules['compound_query_key']] - match_data[self.rules['query_key']] = ",".join([unicode(value) for value in compound_value]) + match_data[self.rules['query_key']] = ",".join([str(value) for value in compound_value]) self.add_match(match_data) @@ -1133,7 +1134,7 @@ def add_aggregation_data(self, payload): We instead want to use all of our SpikeRule.handle_event inherited logic (current/reference) from the aggregation's "value" key to determine spikes from aggregations """ - for timestamp, payload_data in payload.iteritems(): + for timestamp, payload_data in payload.items(): if 'bucket_aggs' in payload_data: self.unwrap_term_buckets(timestamp, payload_data['bucket_aggs']) else: diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 293f0d0f6..72032de1b 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -1,8 +1,5 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function - import argparse import copy import datetime @@ -125,7 +122,7 @@ def test_file(self, conf, args): if args.formatted_output: self.formatted_output['hits'] = num_hits self.formatted_output['days'] = args.days - self.formatted_output['terms'] = terms.keys() + self.formatted_output['terms'] = list(terms.keys()) self.formatted_output['result'] = terms else: print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else '')) @@ -191,7 +188,7 @@ def mock_hits(self, rule, start, end, index, scroll=False): if field != '_id': if not any([re.match(incl.replace('*', '.*'), field) for incl in rule['include']]): fields_to_remove.append(field) - map(doc.pop, fields_to_remove) + list(map(doc.pop, fields_to_remove)) # Separate _source and _id, convert timestamps resp = [{'_source': doc, '_id': doc['_id']} for doc in docs] @@ -211,7 +208,7 @@ def mock_terms(self, rule, start, end, index, key, qk=None, size=None): if qk is None or doc[rule['query_key']] == qk: buckets.setdefault(doc[key], 0) buckets[doc[key]] += 1 - counts = buckets.items() + counts = list(buckets.items()) counts.sort(key=lambda x: x[1], reverse=True) if size: counts = counts[:size] diff --git a/elastalert/util.py b/elastalert/util.py index c53bc3445..2fe4a8e9e 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -8,10 +8,10 @@ import dateutil.parser import pytz -from auth import Auth from six import string_types from . import ElasticSearchClient +from .auth import Auth logging.basicConfig() elastalert_logger = logging.getLogger('elastalert') @@ -27,7 +27,7 @@ def get_module(module_name): base_module = __import__(module_path, globals(), locals(), [module_class]) module = getattr(base_module, module_class) except (ImportError, AttributeError, ValueError) as e: - raise EAException("Could not import module %s: %s" % (module_name, e)), None, sys.exc_info()[2] + raise EAException("Could not import module %s: %s" % (module_name, e)).with_traceback(sys.exc_info()[2]) return module @@ -411,7 +411,7 @@ def parse_deadline(value): def flatten_dict(dct, delim='.', prefix=''): ret = {} - for key, val in dct.items(): + for key, val in list(dct.items()): if type(val) == dict: ret.update(flatten_dict(val, prefix=prefix + key + delim)) else: @@ -442,8 +442,8 @@ def resolve_string(string, match, missing_text=''): string = string.format(**dd_match) break except KeyError as e: - if '{%s}' % e.message not in string: + if '{%s}' % str(e).strip("'") not in string: break - string = string.replace('{%s}' % e.message, '{_missing_value}') + string = string.replace('{%s}' % str(e).strip("'"), '{_missing_value}') return string diff --git a/setup.py b/setup.py index 930b6f6cc..273f01da3 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.2.0b2', + version='0.2.0b3', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 51b7e0f16..549b60624 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- +import base64 import datetime import json import subprocess -from contextlib import nested import mock import pytest @@ -35,7 +35,7 @@ def get_match_str(self, event): def test_basic_match_string(ea): ea.rules[0]['top_count_keys'] = ['username'] match = {'@timestamp': '1918-01-17', 'field': 'value', 'top_events_username': {'bob': 10, 'mallory': 5}} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'anytest' in alert_text assert 'some stuff happened' in alert_text assert 'username' in alert_text @@ -44,32 +44,32 @@ def test_basic_match_string(ea): # Non serializable objects don't cause errors match['non-serializable'] = {open: 10} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) # unicode objects dont cause errors - match['snowman'] = u'☃' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + match['snowman'] = '☃' + alert_text = str(BasicMatchString(ea.rules[0], match)) # Pretty printed objects match.pop('non-serializable') match['object'] = {'this': {'that': [1, 2, "3"]}} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) - assert '"this": {\n "that": [\n 1, \n 2, \n "3"\n ]\n }' in alert_text + alert_text = str(BasicMatchString(ea.rules[0], match)) + assert '"this": {\n "that": [\n 1,\n 2,\n "3"\n ]\n }' in alert_text ea.rules[0]['alert_text'] = 'custom text' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'anytest' not in alert_text ea.rules[0]['alert_text_type'] = 'alert_text_only' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'some stuff happened' not in alert_text assert 'username' not in alert_text assert 'field: value' not in alert_text ea.rules[0]['alert_text_type'] = 'exclude_fields' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'some stuff happened' in alert_text assert 'username' in alert_text @@ -83,8 +83,8 @@ def test_jira_formatted_match_string(ea): expected_alert_text_snippet = '{code}{\n' \ + tab + '"foo": {\n' \ + 2 * tab + '"bar": [\n' \ - + 3 * tab + '"one", \n' \ - + 3 * tab + '2, \n' \ + + 3 * tab + '"one",\n' \ + + 3 * tab + '2,\n' \ + 3 * tab + '"three"\n' \ + 2 * tab + ']\n' \ + tab + '}\n' \ @@ -95,7 +95,7 @@ def test_jira_formatted_match_string(ea): def test_email(): rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test', 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value', - 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': u'☃'} + 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': '☃'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: mock_smtp.return_value = mock.Mock() @@ -158,9 +158,9 @@ def test_email_from_field(): def test_email_with_unicode_strings(): - rule = {'name': 'test alert', 'email': u'testing@test.test', 'from_addr': 'testfrom@test.test', + rule = {'name': 'test alert', 'email': 'testing@test.test', 'from_addr': 'testfrom@test.test', 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value', - 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': u'☃'} + 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': '☃'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: mock_smtp.return_value = mock.Mock() @@ -170,7 +170,7 @@ def test_email_with_unicode_strings(): mock.call().ehlo(), mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), - mock.call().sendmail(mock.ANY, [u'testing@test.test'], mock.ANY), + mock.call().sendmail(mock.ANY, ['testing@test.test'], mock.ANY), mock.call().quit()] assert mock_smtp.mock_calls == expected @@ -329,7 +329,7 @@ def test_email_with_args(): mock_smtp.return_value = mock.Mock() alert = EmailAlerter(rule) - alert.alert([{'test_term': 'test_value', 'test_arg1': 'testing', 'test': {'term': ':)', 'arg3': u'☃'}}]) + alert.alert([{'test_term': 'test_value', 'test_arg1': 'testing', 'test': {'term': ':)', 'arg3': '☃'}}]) expected = [mock.call('localhost'), mock.call().ehlo(), mock.call().has_extn('STARTTLS'), @@ -340,7 +340,7 @@ def test_email_with_args(): body = mock_smtp.mock_calls[4][1][2] # Extract the MIME encoded message body - body_text = body.split('\n\n')[-1][:-1].decode('base64') + body_text = base64.b64decode(body.split('\n\n')[-1][:-1]).decode('utf-8') assert 'testing' in body_text assert '' in body_text @@ -380,9 +380,9 @@ def test_opsgenie_basic(): alert = OpsGenieAlerter(rule) alert.alert([{'@timestamp': '2014-10-31T00:00:00'}]) - print("mock_post: {0}".format(mock_post._mock_call_args_list)) + print(("mock_post: {0}".format(mock_post._mock_call_args_list))) mcal = mock_post._mock_call_args_list - print('mcal: {0}'.format(mcal[0])) + print(('mcal: {0}'.format(mcal[0]))) assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts') assert mock_post.called @@ -406,9 +406,9 @@ def test_opsgenie_frequency(): assert alert.get_info()['recipients'] == rule['opsgenie_recipients'] - print("mock_post: {0}".format(mock_post._mock_call_args_list)) + print(("mock_post: {0}".format(mock_post._mock_call_args_list))) mcal = mock_post._mock_call_args_list - print('mcal: {0}'.format(mcal[0])) + print(('mcal: {0}'.format(mcal[0]))) assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts') assert mock_post.called @@ -478,10 +478,8 @@ def test_jira(): mock_priority = mock.Mock(id='5') - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = [] @@ -511,10 +509,8 @@ def test_jira(): # Search called if jira_bump_tickets rule['jira_bump_tickets'] = True - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -529,10 +525,8 @@ def test_jira(): # Remove a field if jira_ignore_in_title set rule['jira_ignore_in_title'] = 'test_term' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -545,10 +539,8 @@ def test_jira(): assert 'test_value' not in mock_jira.mock_calls[3][1][0] # Issue is still created if search_issues throws an exception - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.side_effect = JIRAError @@ -566,10 +558,8 @@ def test_jira(): # Check ticket is bumped if it is updated 4 days ago mock_issue.fields.updated = str(ts_now() - datetime.timedelta(days=4)) - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -584,10 +574,8 @@ def test_jira(): # Check ticket is bumped is not bumped if ticket is updated right now mock_issue.fields.updated = str(ts_now()) - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -621,10 +609,8 @@ def test_jira(): mock_fields = [ {'name': 'affected user', 'id': 'affected_user_id', 'schema': {'type': 'string'}} ] - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -696,10 +682,8 @@ def test_jira_arbitrary_field_support(): }, ] - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -739,10 +723,8 @@ def test_jira_arbitrary_field_support(): # Reference an arbitrary string field that is not defined on the JIRA server rule['jira_nonexistent_field'] = 'nonexistent field value' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -757,10 +739,8 @@ def test_jira_arbitrary_field_support(): # Reference a watcher that does not exist rule['jira_watchers'] = 'invalid_watcher' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -1567,7 +1547,7 @@ def test_alert_text_kw(ea): 'field': 'field', } match = {'@timestamp': '1918-01-17', 'field': 'value'} - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) body = '{field} at {@timestamp}'.format(**match) assert body in alert_text @@ -1586,7 +1566,7 @@ def test_alert_text_global_substitution(ea): 'abc': 'abc from match', } - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) assert 'Priority: priority from rule' in alert_text assert 'Owner: the owner from rule' in alert_text @@ -1612,7 +1592,7 @@ def test_alert_text_kw_global_substitution(ea): 'abc': 'abc from match', } - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) assert 'Owner: the owner from rule' in alert_text assert 'Foo: foo from rule' in alert_text @@ -1990,7 +1970,7 @@ def test_alerta_no_auth(ea): 'name': 'Test Alerta rule!', 'alerta_api_url': 'http://elastalerthost:8080/api/alert', 'timeframe': datetime.timedelta(hours=1), - 'timestamp_field': u'@timestamp', + 'timestamp_field': '@timestamp', 'alerta_api_skip_ssl': True, 'alerta_attributes_keys': ["hostname", "TimestampEvent", "senderIP"], 'alerta_attributes_values': ["%(key)s", "%(logdate)s", "%(sender_ip)s"], @@ -2007,7 +1987,7 @@ def test_alerta_no_auth(ea): } match = { - u'@timestamp': '2014-10-10T00:00:00', + '@timestamp': '2014-10-10T00:00:00', # 'key': ---- missing field on purpose, to verify that simply the text is left empty # 'logdate': ---- missing field on purpose, to verify that simply the text is left empty 'sender_ip': '1.1.1.1', diff --git a/tests/base_test.py b/tests/base_test.py index 4c6485c39..9b25f6676 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import contextlib import copy import datetime import json @@ -40,7 +39,7 @@ def generate_hits(timestamps, **kwargs): '_source': {'@timestamp': ts}, '_type': 'logs', '_index': 'idx'} - for key, item in kwargs.iteritems(): + for key, item in kwargs.items(): data['_source'][key] = item # emulate process_hits(), add metadata to _source for field in ['_id', '_type', '_index']: @@ -70,7 +69,7 @@ def test_init_rule(ea): # Simulate state of a rule just loaded from a file ea.rules[0]['minimum_starttime'] = datetime.datetime.now() new_rule = copy.copy(ea.rules[0]) - map(new_rule.pop, ['agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime']) + list(map(new_rule.pop, ['agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime'])) # Properties are copied from ea.rules[0] ea.rules[0]['starttime'] = '2014-01-02T00:11:22' @@ -262,8 +261,8 @@ def test_run_rule_calls_garbage_collect(ea): end_time = '2014-09-26T12:00:00Z' ea.buffer_time = datetime.timedelta(hours=1) ea.run_every = datetime.timedelta(hours=1) - with contextlib.nested(mock.patch.object(ea.rules[0]['type'], 'garbage_collect'), - mock.patch.object(ea, 'run_query')) as (mock_gc, mock_get_hits): + with mock.patch.object(ea.rules[0]['type'], 'garbage_collect') as mock_gc, \ + mock.patch.object(ea, 'run_query'): ea.run_rule(ea.rules[0], ts_to_dt(end_time), ts_to_dt(start_time)) # Running ElastAlert every hour for 12 hours, we should see self.garbage_collect called 12 times. @@ -624,12 +623,12 @@ def test_silence(ea): def test_compound_query_key(ea): ea.rules[0]['query_key'] = 'this,that,those' ea.rules[0]['compound_query_key'] = ['this', 'that', 'those'] - hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that=u'☃', those=4) + hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that='☃', those=4) ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) call_args = ea.rules[0]['type'].add_data.call_args_list[0] assert 'this,that,those' in call_args[0][0][0] - assert call_args[0][0][0]['this,that,those'] == u'abc, ☃, 4' + assert call_args[0][0][0]['this,that,those'] == 'abc, ☃, 4' def test_silence_query_key(ea): @@ -754,7 +753,8 @@ def test_realert_with_nested_query_key(ea): def test_count(ea): ea.rules[0]['use_count_query'] = True ea.rules[0]['doc_type'] = 'doctype' - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client'), \ + mock.patch.object(ea, 'get_hits_count') as mock_hits: ea.run_rule(ea.rules[0], END, START) # Assert that es.count is run against every run_every timeframe between START and END @@ -766,8 +766,8 @@ def test_count(ea): end = start + ea.run_every query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end) query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gt'] = dt_to_ts(start) + mock_hits.assert_any_call(mock.ANY, start, end, mock.ANY) start = start + ea.run_every - ea.thread_data.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True) def run_and_assert_segmented_queries(ea, start, end, segment_size): @@ -910,7 +910,8 @@ def test_set_starttime(ea): assert ea.rules[0]['starttime'] == end - ea.run_every # Count query, with previous endtime - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client'), \ + mock.patch.object(ea, 'get_hits_count'): ea.run_rule(ea.rules[0], END, START) ea.set_starttime(ea.rules[0], end) assert ea.rules[0]['starttime'] == END @@ -977,7 +978,7 @@ def test_kibana_dashboard(ea): url = ea.use_kibana_link(ea.rules[0], match) db = json.loads(mock_es.index.call_args_list[-1][1]['body']['dashboard']) found_filters = 0 - for filter_id, filter_dict in db['services']['filter']['list'].items(): + for filter_id, filter_dict in list(db['services']['filter']['list'].items()): if (filter_dict['field'] == 'foo' and filter_dict['query'] == '"cat"') or \ (filter_dict['field'] == 'bar' and filter_dict['query'] == '"dog"'): found_filters += 1 @@ -1113,7 +1114,7 @@ def test_exponential_realert(ea): for args in test_values: ea.silence_cache[ea.rules[0]['name']] = (args[1], args[2]) next_alert, exponent = ea.next_alert_time(ea.rules[0], ea.rules[0]['name'], args[0]) - assert exponent == next_res.next() + assert exponent == next(next_res) def test_wait_until_responsive(ea): diff --git a/tests/conftest.py b/tests/conftest.py index d3f6401ff..457eefee3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -248,7 +248,7 @@ def ea_sixsix(): def environ(): """py.test fixture to get a fresh mutable environment.""" old_env = os.environ - new_env = dict(old_env.items()) + new_env = dict(list(old_env.items())) os.environ = new_env yield os.environ os.environ = old_env diff --git a/tests/create_index_test.py b/tests/create_index_test.py index ba306aee5..47a6247dc 100644 --- a/tests/create_index_test.py +++ b/tests/create_index_test.py @@ -18,36 +18,36 @@ def test_read_default_index_mapping(es_mapping): mapping = elastalert.create_index.read_es_index_mapping(es_mapping) assert es_mapping not in mapping - print(json.dumps(mapping, indent=2)) + print((json.dumps(mapping, indent=2))) @pytest.mark.parametrize('es_mapping', es_mappings) def test_read_es_5_index_mapping(es_mapping): mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 5) assert es_mapping in mapping - print(json.dumps(mapping, indent=2)) + print((json.dumps(mapping, indent=2))) @pytest.mark.parametrize('es_mapping', es_mappings) def test_read_es_6_index_mapping(es_mapping): mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 6) assert es_mapping not in mapping - print(json.dumps(mapping, indent=2)) + print((json.dumps(mapping, indent=2))) def test_read_default_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings() assert len(mappings) == len(es_mappings) - print(json.dumps(mappings, indent=2)) + print((json.dumps(mappings, indent=2))) def test_read_es_5_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings(5) assert len(mappings) == len(es_mappings) - print(json.dumps(mappings, indent=2)) + print((json.dumps(mappings, indent=2))) def test_read_es_6_index_mappings(): mappings = elastalert.create_index.read_es_index_mappings(6) assert len(mappings) == len(es_mappings) - print(json.dumps(mappings, indent=2)) + print((json.dumps(mappings, indent=2))) diff --git a/tests/elasticsearch_test.py b/tests/elasticsearch_test.py index 6b79eb920..997db7e1c 100644 --- a/tests/elasticsearch_test.py +++ b/tests/elasticsearch_test.py @@ -34,9 +34,9 @@ class TestElasticsearch(object): def test_create_indices(self, es_client): elastalert.create_index.create_index_mappings(es_client=es_client, ea_index=test_index) indices_mappings = es_client.indices.get_mapping(test_index + '*') - print('-' * 50) - print(json.dumps(indices_mappings, indent=2)) - print('-' * 50) + print(('-' * 50)) + print((json.dumps(indices_mappings, indent=2))) + print(('-' * 50)) if es_client.is_atleastsix(): assert test_index in indices_mappings assert test_index + '_error' in indices_mappings diff --git a/tests/loaders_test.py b/tests/loaders_test.py index ee1f2407e..509e8d4cb 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -53,7 +53,7 @@ def test_import_rules(): mock_open.return_value = test_rule_copy # Test that type is imported - with mock.patch('__builtin__.__import__') as mock_import: + with mock.patch('builtins.__import__') as mock_import: mock_import.return_value = elastalert.ruletypes rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing.test' @@ -63,7 +63,7 @@ def test_import_rules(): test_rule_copy = copy.deepcopy(test_rule) mock_open.return_value = test_rule_copy test_rule_copy['alert'] = 'testing2.test2.Alerter' - with mock.patch('__builtin__.__import__') as mock_import: + with mock.patch('builtins.__import__') as mock_import: mock_import.return_value = elastalert.alerts rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing2.test2' @@ -212,8 +212,8 @@ def test_load_rules(): with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] + with mock.patch('os.walk') as mock_ls: + mock_ls.return_value = [('', [], ['testrule.yaml'])] rules = load_conf(test_args) rules['rules'] = rules['rules_loader'].load(rules) assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) @@ -238,8 +238,8 @@ def test_load_default_host_port(): with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] + with mock.patch('os.walk') as mock_ls: + mock_ls.return_value = [('', [], ['testrule.yaml'])] rules = load_conf(test_args) rules['rules'] = rules['rules_loader'].load(rules) @@ -325,7 +325,7 @@ def test_load_disabled_rules(): def test_raises_on_missing_config(): optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') test_rule_copy = copy.deepcopy(test_rule) - for key in test_rule_copy.keys(): + for key in list(test_rule_copy.keys()): test_rule_copy = copy.deepcopy(test_rule) test_config_copy = copy.deepcopy(test_config) test_rule_copy.pop(key) @@ -338,12 +338,11 @@ def test_raises_on_missing_config(): mock_conf_open.return_value = test_config_copy with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] + with mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [('', [], ['testrule.yaml'])] with pytest.raises(EAException, message='key %s should be required' % key): rules = load_conf(test_args) rules['rules'] = rules['rules_loader'].load(rules) - print(rules) def test_compound_query_key(): diff --git a/tox.ini b/tox.ini index b8c80496a..6a6efc293 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] project = elastalert -envlist = py27,docs +envlist = py36,docs [testenv] deps = -rrequirements-dev.txt From ef1ed3433f3b8db204db250d7984e1ca6f98e102 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 10 Jun 2019 13:08:49 -0700 Subject: [PATCH 190/264] Changed travis config to py36 --- .travis.yml | 18 +++++++++--------- 1 file changed, 9 insertions(+), 9 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7c8ba2b34..569bf12d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,9 +1,9 @@ language: python python: -- '2.7' +- '3.6' env: - TOXENV=docs -- TOXENV=py27 +- TOXENV=py36 install: - pip install tox - > @@ -19,16 +19,16 @@ script: make test-elasticsearch else make test - fi + fi jobs: include: - stage: 'Elasticsearch test' - env: TOXENV=py27 ES_VERSION=7.0.0-linux-x86_64 - - env: TOXENV=py27 ES_VERSION=6.6.2 - - env: TOXENV=py27 ES_VERSION=6.3.2 - - env: TOXENV=py27 ES_VERSION=6.2.4 - - env: TOXENV=py27 ES_VERSION=6.0.1 - - env: TOXENV=py27 ES_VERSION=5.6.16 + env: TOXENV=py36 ES_VERSION=7.0.0-linux-x86_64 + - env: TOXENV=py36 ES_VERSION=6.6.2 + - env: TOXENV=py36 ES_VERSION=6.3.2 + - env: TOXENV=py36 ES_VERSION=6.2.4 + - env: TOXENV=py36 ES_VERSION=6.0.1 + - env: TOXENV=py36 ES_VERSION=5.6.16 deploy: provider: pypi From d0bd10e67dcdd0d4ecd3e3a1da5d03fa32cb0cc9 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Tue, 9 Jul 2019 23:45:33 +0200 Subject: [PATCH 191/264] Show disabled rules --- elastalert/elastalert.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index e08b89503..a552be238 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1108,6 +1108,9 @@ def start(self): if next_run < datetime.datetime.utcnow(): continue + #Show disabled rules + self.show_disabled_rules() + # Wait before querying again sleep_duration = total_seconds(next_run - datetime.datetime.utcnow()) self.sleep_for(sleep_duration) @@ -1200,6 +1203,14 @@ def stop(self): """ Stop an ElastAlert runner that's been started """ self.running = False + def get_disabled_rules(self): + """ Return disabled rules """ + return self.disabled_rules + + def show_disabled_rules(self): + """ Show disabled rules """ + elastalert_logger.info("Disabled rules are: %s" % (str(self.get_disabled_rules()))) + def sleep_for(self, duration): """ Sleep for a set duration """ elastalert_logger.info("Sleeping for %s seconds" % (duration)) From cab5bc8e7149bc3ec041ff76611c13747b2ba525 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Wed, 10 Jul 2019 00:14:55 +0200 Subject: [PATCH 192/264] E265 block comment should start with '# ' --- elastalert/elastalert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index a552be238..bbba7a669 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1108,7 +1108,7 @@ def start(self): if next_run < datetime.datetime.utcnow(): continue - #Show disabled rules + # Show disabled rules self.show_disabled_rules() # Wait before querying again From 34d950103f1d2d5a7311696ce0a794606c5ff103 Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Wed, 10 Jul 2019 09:53:38 +0200 Subject: [PATCH 193/264] Config param to control the output of disabled_rules --- docs/source/elastalert.rst | 2 ++ elastalert/config.py | 1 + elastalert/elastalert.py | 7 ++++--- 3 files changed, 7 insertions(+), 3 deletions(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 55aa0057b..12a07fe31 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -167,6 +167,8 @@ from that time, unless it is older than ``old_query_limit``, in which case it wi will upload a traceback message to ``elastalert_metadata`` and if ``notify_email`` is set, send an email notification. The rule will no longer be run until either ElastAlert restarts or the rule file has been modified. This defaults to True. +``show_disabled_rules``: If true, ElastAlert show the disable rules' list when finishes the execution. This defaults to True. + ``notify_email``: An email address, or list of email addresses, to which notification emails will be sent. Currently, only an uncaught exception will send a notification email. The from address, SMTP host, and reply-to header can be set using ``from_addr``, ``smtp_host``, and ``email_reply_to`` options, respectively. By default, no emails will be sent. diff --git a/elastalert/config.py b/elastalert/config.py index 7b7fc30ba..25b570a52 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -478,6 +478,7 @@ def load_rules(args): conf.setdefault('max_scrolling_count', 0) conf.setdefault('disable_rules_on_error', True) conf.setdefault('scan_subdirectories', True) + conf.setdefault('show_disabled_rules', True) # Convert run_every, buffer_time into a timedelta object try: diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index bbba7a669..8b23e403d 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1109,7 +1109,8 @@ def start(self): continue # Show disabled rules - self.show_disabled_rules() + if self.conf['show_disabled_rules']: + self.print_disabled_rules() # Wait before querying again sleep_duration = total_seconds(next_run - datetime.datetime.utcnow()) @@ -1207,8 +1208,8 @@ def get_disabled_rules(self): """ Return disabled rules """ return self.disabled_rules - def show_disabled_rules(self): - """ Show disabled rules """ + def print_disabled_rules(self): + """ Show disabled rules via info logger """ elastalert_logger.info("Disabled rules are: %s" % (str(self.get_disabled_rules()))) def sleep_for(self, duration): From 7fb3485e03f39adcb686c7fe2d06aed4fb4faa8c Mon Sep 17 00:00:00 2001 From: Alvaro Olmedo Date: Wed, 10 Jul 2019 10:14:00 +0200 Subject: [PATCH 194/264] Extract conf param directly in elastalert class and remove print function --- elastalert/config.py | 1 - elastalert/elastalert.py | 9 +++------ 2 files changed, 3 insertions(+), 7 deletions(-) diff --git a/elastalert/config.py b/elastalert/config.py index 25b570a52..7b7fc30ba 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -478,7 +478,6 @@ def load_rules(args): conf.setdefault('max_scrolling_count', 0) conf.setdefault('disable_rules_on_error', True) conf.setdefault('scan_subdirectories', True) - conf.setdefault('show_disabled_rules', True) # Convert run_every, buffer_time into a timedelta object try: diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 8b23e403d..a8b639c17 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -132,6 +132,7 @@ def __init__(self, args): self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) self.string_multi_field_name = self.conf.get('string_multi_field_name', False) self.add_metadata_alert = self.conf.get('add_metadata_alert', False) + self.show_disabled_rules = self.conf.get('show_disabled_rules', True) self.writeback_es = elasticsearch_client(self.conf) @@ -1109,8 +1110,8 @@ def start(self): continue # Show disabled rules - if self.conf['show_disabled_rules']: - self.print_disabled_rules() + if self.show_disabled_rules: + elastalert_logger.info("Disabled rules are: %s" % (str(self.get_disabled_rules()))) # Wait before querying again sleep_duration = total_seconds(next_run - datetime.datetime.utcnow()) @@ -1208,10 +1209,6 @@ def get_disabled_rules(self): """ Return disabled rules """ return self.disabled_rules - def print_disabled_rules(self): - """ Show disabled rules via info logger """ - elastalert_logger.info("Disabled rules are: %s" % (str(self.get_disabled_rules()))) - def sleep_for(self, duration): """ Sleep for a set duration """ elastalert_logger.info("Sleeping for %s seconds" % (duration)) From 7980f9b4c3b03458a403fa2acbc1dd3393555ebb Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 10 Jul 2019 10:12:55 -0700 Subject: [PATCH 195/264] Only print disabled rule names --- elastalert/elastalert.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 1fc1593a5..d82118935 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -43,12 +43,12 @@ from util import replace_dots_in_field_names from util import seconds from util import set_es_key +from util import should_scrolling_continue from util import total_seconds from util import ts_add from util import ts_now from util import ts_to_dt from util import unix_to_dt -from util import should_scrolling_continue class ElastAlerter(object): @@ -1209,7 +1209,7 @@ def stop(self): def get_disabled_rules(self): """ Return disabled rules """ - return self.disabled_rules + return [rule['name'] for rule in self.disabled_rules] def sleep_for(self, duration): """ Sleep for a set duration """ From 76ab593aeb57355d7084314804d7e89eb2449157 Mon Sep 17 00:00:00 2001 From: Valerio Sabelli Date: Fri, 12 Jul 2019 12:14:19 +0200 Subject: [PATCH 196/264] When user chooses a compound query key, the split is not performed correctly due to a blank in text. Therefore, a malformed query is performed to Elasticsearch. This reflects in results not available when using top_count_keys directive in rule file. --- elastalert/elastalert.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index d82118935..17d445e5d 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -439,7 +439,8 @@ def get_hits_count(self, rule, starttime, endtime, index): def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=None): rule_filter = copy.copy(rule['filter']) if qk: - qk_list = qk.split(", ") + qk_tmp = qk.replace(" ", "") + qk_list = qk.split(",") end = None if rule['five']: end = '.keyword' From b341220528ffe80f1fd513b9d3391a1044cb4cc3 Mon Sep 17 00:00:00 2001 From: Valerio Sabelli Date: Fri, 12 Jul 2019 12:36:30 +0200 Subject: [PATCH 197/264] Added support to use match fields to populate opsgenie_alias in integration with OpsGenie (this was already available for opsgenie_alias). --- elastalert/opsgenie.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index 6a6e7fda7..ffe9c567a 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -81,7 +81,10 @@ def alert(self, matches): post['teams'] = [{'name': r, 'type': 'team'} for r in self.teams] post['description'] = body post['source'] = 'ElastAlert' - post['tags'] = self.tags + + if self.tags is not None: + post['tags'] = self.tags.format(**matches[0]) + if self.priority and self.priority not in ('P1', 'P2', 'P3', 'P4', 'P5'): logging.warn("Priority level does not appear to be specified correctly. \ Please make sure to set it to a value between P1 and P5") From 8abff23afcd75234d2d4c95f99cc630cc57b1c31 Mon Sep 17 00:00:00 2001 From: Valerio Sabelli Date: Fri, 12 Jul 2019 12:46:47 +0200 Subject: [PATCH 198/264] Removed unused variable (to fix error in Travis CI). --- elastalert/elastalert.py | 1 - 1 file changed, 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 17d445e5d..11e037a6e 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -439,7 +439,6 @@ def get_hits_count(self, rule, starttime, endtime, index): def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=None): rule_filter = copy.copy(rule['filter']) if qk: - qk_tmp = qk.replace(" ", "") qk_list = qk.split(",") end = None if rule['five']: From 386708e223329f9ececa9e0f2df4d3b60acad3c6 Mon Sep 17 00:00:00 2001 From: Valerio Sabelli Date: Fri, 12 Jul 2019 12:56:30 +0200 Subject: [PATCH 199/264] Fix for variable type (to solve Travis CI problem). --- elastalert/opsgenie.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index ffe9c567a..61b4c1d20 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -81,10 +81,14 @@ def alert(self, matches): post['teams'] = [{'name': r, 'type': 'team'} for r in self.teams] post['description'] = body post['source'] = 'ElastAlert' - + for tag in self.tags: if self.tags is not None: post['tags'] = self.tags.format(**matches[0]) + for i, tag in enumerate(self.tags): + self.tags[i] = tag.format(**matches[0]) + post['tags'] = self.tags + if self.priority and self.priority not in ('P1', 'P2', 'P3', 'P4', 'P5'): logging.warn("Priority level does not appear to be specified correctly. \ Please make sure to set it to a value between P1 and P5") From 8219e090cb852e11a691eda8c2d450dff5636334 Mon Sep 17 00:00:00 2001 From: Valerio Sabelli Date: Fri, 12 Jul 2019 12:59:25 +0200 Subject: [PATCH 200/264] Fix paste error (for Travis CI problem). --- elastalert/opsgenie.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index 61b4c1d20..30f5302bc 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -81,9 +81,6 @@ def alert(self, matches): post['teams'] = [{'name': r, 'type': 'team'} for r in self.teams] post['description'] = body post['source'] = 'ElastAlert' - for tag in self.tags: - if self.tags is not None: - post['tags'] = self.tags.format(**matches[0]) for i, tag in enumerate(self.tags): self.tags[i] = tag.format(**matches[0]) From df4ef0c06e8966abd8dee3e52af02d695103449c Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 15 Jul 2019 14:58:25 -0700 Subject: [PATCH 201/264] Fixed elasticsearch tests --- tests/conftest.py | 4 ++-- tests/elasticsearch_test.py | 8 ++++++-- 2 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 457eefee3..2b547ba41 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -184,9 +184,9 @@ def ea(): ea.rules[0]['type'] = mock_ruletype() ea.rules[0]['alert'] = [mock_alert()] ea.writeback_es = mock_es_client() - ea.writeback_es.search.return_value = {'hits': {'hits': []}} + ea.writeback_es.search.return_value = {'hits': {'hits': []}, 'total': 0} ea.writeback_es.deprecated_search.return_value = {'hits': {'hits': []}} - ea.writeback_es.index.return_value = {'_id': 'ABCD'} + ea.writeback_es.index.return_value = {'_id': 'ABCD', 'created': True} ea.current_es = mock_es_client('', '') ea.thread_data.current_es = ea.current_es ea.thread_data.num_hits = 0 diff --git a/tests/elasticsearch_test.py b/tests/elasticsearch_test.py index 997db7e1c..308356c25 100644 --- a/tests/elasticsearch_test.py +++ b/tests/elasticsearch_test.py @@ -51,7 +51,7 @@ def test_create_indices(self, es_client): assert 'past_elastalert' in indices_mappings[test_index]['mappings'] @pytest.mark.usefixtures("ea") - def test_aggregated_alert(self, ea): # noqa: F811 + def test_aggregated_alert(self, ea, es_client): # noqa: F811 match_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( days=1) ea.rules[0]['aggregate_by_match_time'] = True @@ -59,6 +59,7 @@ def test_aggregated_alert(self, ea): # noqa: F811 'num_hits': 0, 'num_matches': 3 } + ea.writeback_es = es_client res = ea.add_aggregated_alert(match, ea.rules[0]) if ea.writeback_es.is_atleastsix(): assert res['result'] == 'created' @@ -70,9 +71,10 @@ def test_aggregated_alert(self, ea): # noqa: F811 assert ea.find_pending_aggregate_alert(ea.rules[0]) @pytest.mark.usefixtures("ea") - def test_silenced(self, ea): # noqa: F811 + def test_silenced(self, ea, es_client): # noqa: F811 until_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( days=1) + ea.writeback_es = es_client res = ea.set_realert(ea.rules[0]['name'], until_timestamp, 0) if ea.writeback_es.is_atleastsix(): assert res['result'] == 'created' @@ -94,5 +96,7 @@ def test_get_hits(self, ea, es_client): # noqa: F811 ea.rules[0]['five'] = True else: ea.rules[0]['five'] = False + ea.thread_data.current_es = ea.current_es hits = ea.get_hits(ea.rules[0], start, end, test_index) + assert isinstance(hits, list) From ecf40ad31530c34c522d2f8162f6df8f4b32fb1c Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 15 Jul 2019 16:58:23 -0700 Subject: [PATCH 202/264] Fixed tests by setting some default args --- elastalert/test_rule.py | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 72032de1b..629ce5a06 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -402,6 +402,13 @@ def run_rule_test(self): overwrites = { 'rules_loader': 'file', } + + # Set arguments that ElastAlerter needs + args.verbose = args.alert + args.debug = not args.verbose + args.es_debug = False + args.es_debug_trace = False + conf = load_conf(args, defaults, overwrites) rule_yaml = conf['rules_loader'].get_yaml(args.file) conf['rules_loader'].load_options(rule_yaml, conf, args.file) From 4fca69832442b628b45055905de0a581d01c5138 Mon Sep 17 00:00:00 2001 From: Abhishek Jaisingh Date: Wed, 17 Jul 2019 19:08:26 +0530 Subject: [PATCH 203/264] Fix Invalid Example Rule File Add `timeframe` key in `example_spike_single_metric_agg.yaml`, which is specified to be `required` in _schema.yaml_ --- example_rules/example_spike_single_metric_agg.yaml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/example_rules/example_spike_single_metric_agg.yaml b/example_rules/example_spike_single_metric_agg.yaml index 9248f8fb2..b26ade15a 100644 --- a/example_rules/example_spike_single_metric_agg.yaml +++ b/example_rules/example_spike_single_metric_agg.yaml @@ -6,6 +6,9 @@ type: spike_aggregation index: metricbeat-* +timeframe: + hours: 4 + buffer_time: hours: 1 From aaeec1bb3acaf256f8e025a21d9ec408ca03cc6a Mon Sep 17 00:00:00 2001 From: Asher Foa <1268088+asherf@users.noreply.github.com> Date: Tue, 23 Jul 2019 16:27:01 -0700 Subject: [PATCH 204/264] Remove dead links. --- README.md | 2 -- 1 file changed, 2 deletions(-) diff --git a/README.md b/README.md index 8757290f6..0557c2744 100644 --- a/README.md +++ b/README.md @@ -1,8 +1,6 @@ Note: If you're using Elasticsearch 7, you'll need to install a beta release of Elastalert: `pip install "elastalert>=0.2.0b"` -[![Stories in Ready](https://badge.waffle.io/Yelp/elastalert.png?label=ready&title=Ready)](https://waffle.io/Yelp/elastalert) -[![Stories in In Progress](https://badge.waffle.io/Yelp/elastalert.png?label=in%20progress&title=In%20Progress)](https://waffle.io/Yelp/elastalert) [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From b65b9718075677b313e42561eab2eb524534ea01 Mon Sep 17 00:00:00 2001 From: Ruben van Vreeland Date: Wed, 24 Jul 2019 18:10:32 +0200 Subject: [PATCH 205/264] Adds to the elasticsearch mapping a dynaic_templates configuration. Now, aggregations can be performed on alerts, for example in visualisations. This also enables ElastAlert to run over ElastAlert alerts. --- elastalert/es_mappings/6/elastalert.json | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/elastalert/es_mappings/6/elastalert.json b/elastalert/es_mappings/6/elastalert.json index ecc30c8ed..645a67762 100644 --- a/elastalert/es_mappings/6/elastalert.json +++ b/elastalert/es_mappings/6/elastalert.json @@ -1,4 +1,17 @@ { + "numeric_detection": true, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], "properties": { "rule_name": { "type": "keyword" @@ -16,8 +29,7 @@ "format": "dateOptionalTime" }, "match_body": { - "type": "object", - "enabled": "false" + "type": "object" }, "aggregate_id": { "type": "keyword" From 4b927b95e263a21b8654a8d18a2a4a3a8a37d551 Mon Sep 17 00:00:00 2001 From: Ruben van Vreeland Date: Thu, 25 Jul 2019 13:09:28 +0200 Subject: [PATCH 206/264] Enables querying the source index and correlate with ElastAlert. Add support to put the match in the root ElastAlert document. For example, if your query_key is source.ip, source.ip will also be present in the ElastAlert index. --- elastalert/elastalert.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 11e037a6e..b68fa671b 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1460,6 +1460,9 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No 'alert_time': alert_time } + if rule.get('include_match_in_root'): + body.update({k: v for k, v in match.items() if not k.startswith('_')}) + if self.add_metadata_alert: body['category'] = rule['category'] body['description'] = rule['description'] From bb248f9b550fa8b3ef8d999e0b1f00cf36bed7dd Mon Sep 17 00:00:00 2001 From: Ruben van Vreeland Date: Thu, 25 Jul 2019 13:10:23 +0200 Subject: [PATCH 207/264] Add sample rules SSH usees include_match_in_root to give an example of the feature. Repeat offender is an example of a second order rule. --- example_rules/ssh-repeat-offender.yaml | 61 ++++++++++++++++++++++++ example_rules/ssh.yaml | 64 ++++++++++++++++++++++++++ 2 files changed, 125 insertions(+) create mode 100644 example_rules/ssh-repeat-offender.yaml create mode 100644 example_rules/ssh.yaml diff --git a/example_rules/ssh-repeat-offender.yaml b/example_rules/ssh-repeat-offender.yaml new file mode 100644 index 000000000..27a439fcd --- /dev/null +++ b/example_rules/ssh-repeat-offender.yaml @@ -0,0 +1,61 @@ +# Rule name, must be unique +name: SSH abuse - reapeat offender + +# Alert on x events in y seconds +type: frequency + +# Alert when this many documents matching the query occur within a timeframe +num_events: 2 + +# num_events must occur within this amount of time to trigger an alert +timeframe: + weeks: 1 + +# A list of elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: + - term: + rule_name: "SSH abuse" + +index: elastalert + +# When the attacker continues, send a new alert after x minutes +realert: + weeks: 4 + +query_key: + - match_body.source.ip + +include: + - match_body.host.hostname + - match_body.user.name + - match_body.source.ip + +alert_subject: "SSH abuse (repeat offender) on <{}> | <{}|Show Dashboard>" +alert_subject_args: + - match_body.host.hostname + - kibana_link + +alert_text: |- + An reapeat offender has been active on {}. + + IP: {} + User: {} +alert_text_args: + - match_body.host.hostname + - match_body.user.name + - match_body.source.ip + +# The alert is use when a match is found +alert: + - slack + +slack_webhook_url: "https://hooks.slack.com/services/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" +slack_username_override: "ElastAlert" + +# Alert body only cointains a title and text +alert_text_type: alert_text_only + +# Link to BitSensor Kibana Dashboard +use_kibana4_dashboard: "https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb" diff --git a/example_rules/ssh.yaml b/example_rules/ssh.yaml new file mode 100644 index 000000000..7af890784 --- /dev/null +++ b/example_rules/ssh.yaml @@ -0,0 +1,64 @@ +# Rule name, must be unique + name: SSH abuse (ElastAlert 3.0.1) - 2 + +# Alert on x events in y seconds +type: frequency + +# Alert when this many documents matching the query occur within a timeframe +num_events: 20 + +# num_events must occur within this amount of time to trigger an alert +timeframe: + minutes: 60 + +# A list of elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: +- query: + query_string: + query: "event.type:authentication_failure" + +index: auditbeat-* + +# When the attacker continues, send a new alert after x minutes +realert: + minutes: 1 + +query_key: + - source.ip + +include: + - host.hostname + - user.name + - source.ip + +include_match_in_root: true + +alert_subject: "SSH abuse on <{}> | <{}|Show Dashboard>" +alert_subject_args: + - host.hostname + - kibana_link + +alert_text: |- + An attack on {} is detected. + The attacker looks like: + User: {} + IP: {} +alert_text_args: + - host.hostname + - user.name + - source.ip + +# The alert is use when a match is found +alert: + - debug + +slack_webhook_url: "https://hooks.slack.com/services/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" +slack_username_override: "ElastAlert" + +# Alert body only cointains a title and text +alert_text_type: alert_text_only + +# Link to BitSensor Kibana Dashboard +use_kibana4_dashboard: "https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb" From 70dab8f49b6ea259d2fd26359d6f0533d394e902 Mon Sep 17 00:00:00 2001 From: Luke Watson Date: Mon, 29 Jul 2019 11:06:50 -0400 Subject: [PATCH 208/264] Add "http_post_headers" to documentation --- docs/source/ruletypes.rst | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index f8452dcdd..ba43a187f 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2037,6 +2037,8 @@ Optional: ``http_post_static_payload``: Key:value pairs of static parameters to be sent, along with the Elasticsearch results. Put your authentication or other information here. +``http_post_headers``: Key:value pairs of headers to be sent as part of the request. + ``http_post_proxy``: URL of proxy, if required. ``http_post_all_values``: Boolean of whether or not to include every key value pair from the match in addition to those in http_post_payload and http_post_static_payload. Defaults to True if http_post_payload is not specified, otherwise False. @@ -2051,6 +2053,8 @@ Example usage:: ip: clientip http_post_static_payload: apikey: abc123 + http_post_headers: + authorization: Basic 123dr3234 Alerter From f6ce4223400cb1f880d5c390515c77deb4b232b2 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 11:26:41 -0700 Subject: [PATCH 209/264] Fixed deprecated search --- elastalert/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 05d9ef7ff..d0d850aa0 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -245,4 +245,4 @@ def deprecated_search(self, index=None, doc_type=None, body=None, params=None): index = "_all" return self.transport.perform_request( "GET", _make_path(index, doc_type, "_search"), params=params, body=body - ) + )[1] From 92d7327668d84bf6dc1143f01af3c4b5afd9a338 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 11:30:24 -0700 Subject: [PATCH 210/264] Remove doc_type warning --- elastalert/__init__.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index d0d850aa0..30e3ead9a 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- import copy -import logging from elasticsearch import Elasticsearch from elasticsearch import RequestsHttpConnection @@ -235,8 +234,6 @@ def deprecated_search(self, index=None, doc_type=None, body=None, params=None): :arg version: Specify whether to return document version as part of a hit """ - logging.warning( - 'doc_type has been deprecated since elasticsearch version 6 and will be completely removed in 8') # from is a reserved word so it cannot be used, use from_ instead if "from_" in params: params["from"] = params.pop("from_") From 6950ea80104c190703eaa8fa19b4add558ce6717 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 12:07:55 -0700 Subject: [PATCH 211/264] Fix deprecated_search return type for all versions --- elastalert/__init__.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 30e3ead9a..daad8eb29 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -240,6 +240,9 @@ def deprecated_search(self, index=None, doc_type=None, body=None, params=None): if not index: index = "_all" - return self.transport.perform_request( + res = self.transport.perform_request( "GET", _make_path(index, doc_type, "_search"), params=params, body=body - )[1] + ) + if type(res) == list: + return res[1] + return res From 007f4a50cc18052bc8998a18d7950525833aa673 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 16:04:59 -0700 Subject: [PATCH 212/264] Add retries and caching to ES info --- elastalert/__init__.py | 9 ++++++++- elastalert/elastalert.py | 5 ++++- tests/base_test.py | 2 +- 3 files changed, 13 insertions(+), 3 deletions(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index daad8eb29..86b2a3735 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -1,10 +1,12 @@ # -*- coding: utf-8 -*- import copy +import time from elasticsearch import Elasticsearch from elasticsearch import RequestsHttpConnection from elasticsearch.client import _make_path from elasticsearch.client import query_params +from elasticsearch.exceptions import TransportError class ElasticSearchClient(Elasticsearch): @@ -42,7 +44,12 @@ def es_version(self): Returns the reported version from the Elasticsearch server. """ if self._es_version is None: - self._es_version = self.info()['version']['number'] + for retry in range(3): + try: + self._es_version = self.info()['version']['number'] + break + except TransportError: + time.sleep(3) return self._es_version def is_atleastfive(self): diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 98e9dc8f0..4ac9e1e56 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -105,6 +105,7 @@ def parse_args(self, args): self.args = parser.parse_args(args) def __init__(self, args): + self.es_clients = {} self.parse_args(args) self.debug = self.args.debug self.verbose = self.args.verbose @@ -843,7 +844,7 @@ def run_rule(self, rule, endtime, starttime=None): :return: The number of matches that the rule produced. """ run_start = time.time() - self.thread_data.current_es = elasticsearch_client(rule) + self.thread_data.current_es = self.es_clients.setdefault(rule['name'], elasticsearch_client(rule)) # If there are pending aggregate matches, try processing them for x in range(len(rule['agg_matches'])): @@ -1115,6 +1116,8 @@ def load_rule_changes(self): continue if self.init_rule(new_rule): elastalert_logger.info('Loaded new rule %s' % (rule_file)) + if new_rule['name'] in self.es_clients: + self.es_clients.pop(new_rule['name']) self.rules.append(new_rule) self.rule_hashes = new_rule_hashes diff --git a/tests/base_test.py b/tests/base_test.py index 9b25f6676..15474c690 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -516,7 +516,7 @@ def test_agg_no_writeback_connectivity(ea): ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.add_aggregated_alert = mock.Mock() - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch.object(ea, 'run_query'): ea.run_rule(ea.rules[0], END, START) ea.add_aggregated_alert.assert_any_call({'@timestamp': hit1, 'num_hits': 0, 'num_matches': 3}, ea.rules[0]) From e79eb4041a6732c190dfe3d1bf4a19d2388cc15c Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 29 Jul 2019 17:17:31 -0700 Subject: [PATCH 213/264] Reraise error if retries fail (get_info) --- elastalert/__init__.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index 86b2a3735..f2b868c81 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -49,6 +49,8 @@ def es_version(self): self._es_version = self.info()['version']['number'] break except TransportError: + if retry == 2: + raise time.sleep(3) return self._es_version From 3d103eef83e3d25f9dc353c36825a98becc380a3 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 1 Aug 2019 16:23:05 -0700 Subject: [PATCH 214/264] Update elastalert/test_rule.py Co-Authored-By: Reid Miller --- elastalert/test_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 629ce5a06..627dfe82c 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -405,7 +405,7 @@ def run_rule_test(self): # Set arguments that ElastAlerter needs args.verbose = args.alert - args.debug = not args.verbose + args.debug = not args.alert args.es_debug = False args.es_debug_trace = False From 3555faee35d95ec847a781b844e10ef5d5c68d43 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 2 Aug 2019 13:37:25 -0700 Subject: [PATCH 215/264] Fix deprecated_Search --- elastalert/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/__init__.py b/elastalert/__init__.py index f2b868c81..55bfdb32f 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -252,6 +252,6 @@ def deprecated_search(self, index=None, doc_type=None, body=None, params=None): res = self.transport.perform_request( "GET", _make_path(index, doc_type, "_search"), params=params, body=body ) - if type(res) == list: + if type(res) == list or type(res) == tuple: return res[1] return res From 15ed5772331765c529b7c5475d5086f296792aff Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 7 Aug 2019 13:47:38 -0700 Subject: [PATCH 216/264] Version 0.2.0 --- setup.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/setup.py b/setup.py index 273f01da3..c71af3d30 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,14 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.2.0b3', + version='0.2.0', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', setup_requires='setuptools', license='Copyright 2014 Yelp', classifiers=[ - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', ], From f30de3915adf10c9112c54651f69bcbf5579c6ab Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 7 Aug 2019 13:59:26 -0700 Subject: [PATCH 217/264] Remove ES7 note --- README.md | 3 --- 1 file changed, 3 deletions(-) diff --git a/README.md b/README.md index 0557c2744..9ce2ad445 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,3 @@ -Note: If you're using Elasticsearch 7, you'll need to install a beta release of Elastalert: `pip install "elastalert>=0.2.0b"` - - [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) From 2ba314ecb3ac236493e512064105fd247e70a382 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 8 Aug 2019 14:49:54 -0700 Subject: [PATCH 218/264] Fix a bug introduced in 2.0 merge --- elastalert/elastalert.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 30f32c03e..cafb6ef57 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -657,7 +657,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): if 'scroll_id' in rule: scroll_id = rule.pop('scroll_id') - self.current_es.clear_scroll(scroll_id=scroll_id) + self.thread_data.current_es.clear_scroll(scroll_id=scroll_id) return True From 4dd50dc3f1147d692b66ca05c3ec8642a70e9af5 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 8 Aug 2019 14:56:49 -0700 Subject: [PATCH 219/264] Version 0.2.1 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index c71af3d30..412d53b1e 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.2.0', + version='0.2.1', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From a5322f25b80828cc2183970c9c9f81f32867ce3b Mon Sep 17 00:00:00 2001 From: Abhishek Jaisingh Date: Mon, 12 Aug 2019 20:33:35 +0530 Subject: [PATCH 220/264] Catch Scroll Clear NotFound Error --- elastalert/elastalert.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index cafb6ef57..5436ada18 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -24,6 +24,7 @@ from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException from elasticsearch.exceptions import TransportError +from elasticsearch.exceptions import NotFoundError from . import kibana from .alerts import DebugAlerter @@ -657,7 +658,10 @@ def run_query(self, rule, start=None, end=None, scroll=False): if 'scroll_id' in rule: scroll_id = rule.pop('scroll_id') - self.thread_data.current_es.clear_scroll(scroll_id=scroll_id) + try: + self.thread_data.current_es.clear_scroll(scroll_id=scroll_id) + except NotFoundError: + pass return True From 1df5e8feff119ded9b975e8cfab71820c0628f6e Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 12 Aug 2019 12:41:16 -0700 Subject: [PATCH 221/264] update requirements, docs, changelog --- README.md | 2 ++ changelog.md | 17 +++++++++++++++++ docs/source/running_elastalert.rst | 2 +- requirements.txt | 2 +- 4 files changed, 21 insertions(+), 2 deletions(-) diff --git a/README.md b/README.md index 9ce2ad445..94a091aef 100644 --- a/README.md +++ b/README.md @@ -1,3 +1,5 @@ +Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported. + [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) diff --git a/changelog.md b/changelog.md index 30b52f601..fe30b573c 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,22 @@ # Change Log +# v0.2.1 + +### Fixed +- Fixed an AttributeError introduced in 0.2.0 + +# v0.2.0 + +- Switched to Python 3 + +### Added +- Add rule loader class for customized rule loading +- Added thread based rules and limit_execution +- Run_every can now be customized per rule + +### Fixed +- Various small fixes + # v0.1.39 ### Added diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 09e307c24..7fdf1eeba 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -8,7 +8,7 @@ Requirements - Elasticsearch - ISO8601 or Unix timestamped data -- Python 2.7 +- Python 3.6 - pip, see requirements.txt - Packages on Ubuntu 14.x: python-pip python-dev libffi-dev libssl-dev diff --git a/requirements.txt b/requirements.txt index e70af5896..3e5cbed90 100644 --- a/requirements.txt +++ b/requirements.txt @@ -17,7 +17,7 @@ PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 python-magic>=0.4.15 python-magic>=0.4.15 -PyYAML>=3.12 +PyYAML>=5.1 requests>=2.0.0 stomp.py>=4.1.17 texttable>=0.8.8 From 3affdd7ebf2b6d82c5cfcad0092df97dc27cd8b1 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 13 Aug 2019 10:21:32 -0700 Subject: [PATCH 222/264] Fixed config.yaml default regression --- elastalert/config.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/elastalert/config.py b/elastalert/config.py index c912122e0..5ae9a26e6 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -41,7 +41,13 @@ def load_conf(args, defaults=None, overwrites=None): :return: The global configuration, a dictionary. """ filename = args.config - conf = yaml_loader(filename) + if filename: + conf = yaml_loader(filename) + else: + try: + conf = yaml_loader('config.yaml') + except FileNotFoundError: + raise EAException('No --config or config.yaml found') # init logging from config and set log levels according to command line options configure_logging(args, conf) From 1d3c1021c6c06b08f8483e2e0eb835bbc61ac21c Mon Sep 17 00:00:00 2001 From: Josh Brower Date: Wed, 21 Aug 2019 09:56:11 -0400 Subject: [PATCH 223/264] Remove hive_port When using TheHive in a configuration that is not top-level (`https://IP/thehive/`), the current code does not work, as it concatenates `hive_host` & `hive_port` which creates `https://IP/thehive/:443` This tweak removes the need to explicitly specify a port and instead specify only `hive_host`. A couple examples: `hive_host: https://192.168.15.23:8080` `hive_host: https://192.168.15.23/thehive/` `hive_host: https://192.168.15.23:8080/thehive/` This tweak should be considered a breaking change, as current users will need to update their `hive_connection` configuration if `hive_url` does not contain the required port. --- elastalert/alerts.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3ee892d4..927ced575 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2111,7 +2111,7 @@ def alert(self, matches): connection_details = self.rule['hive_connection'] api = TheHiveApi( - '{hive_host}:{hive_port}'.format(**connection_details), + connection_details.get('hive_host'), connection_details.get('hive_apikey', ''), proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), cert=connection_details.get('hive_verify', False)) From d75a07f38eacfbef104f48be913f1f77bf79aa2b Mon Sep 17 00:00:00 2001 From: Abhishek Jaisingh Date: Fri, 23 Aug 2019 13:07:04 +0530 Subject: [PATCH 224/264] Fix Running ElastAlert instructions in README Fix #2417 , this comes after the switch to Python3, 3 months back --- README.md | 17 ++++++++++++++++- 1 file changed, 16 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 94a091aef..99acc02e7 100644 --- a/README.md +++ b/README.md @@ -71,8 +71,23 @@ In addition to this basic usage, there are many other features that make alerts To get started, check out `Running ElastAlert For The First Time` in the [documentation](http://elastalert.readthedocs.org). ## Running ElastAlert +You can either install the latest released version of ElastAlert using pip: -``$ python elastalert/elastalert.py [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` +```pip install elastalert``` + +or you can clone the ElastAlert repository for the most recent changes: + +```git clone https://github.com/Yelp/elastalert.git``` + +Install the module: + +```pip install "setuptools>=11.3"``` + +```python setup.py install``` + +The following invocation can be used to run ElastAlert after installing + +``$ elastalert [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` ``--debug`` will print additional information to the screen as well as suppresses alerts and instead prints the alert body. Not compatible with `--verbose`. From 0022a01f4cca0a83d4f26eed6ef137fcdae65f55 Mon Sep 17 00:00:00 2001 From: admin Date: Tue, 3 Sep 2019 09:14:56 -0400 Subject: [PATCH 225/264] convert str to byte before pipe it to command --- elastalert/alerts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3ee892d4..84b0ae482 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -918,10 +918,10 @@ def alert(self, matches): if self.rule.get('pipe_match_json'): match_json = json.dumps(matches, cls=DateTimeEncoder) + '\n' - stdout, stderr = subp.communicate(input=match_json) + stdout, stderr = subp.communicate(input=match_json.encode()) elif self.rule.get('pipe_alert_text'): alert_text = self.create_alert_body(matches) - stdout, stderr = subp.communicate(input=alert_text) + stdout, stderr = subp.communicate(input=alert_text.encode()) if self.rule.get("fail_on_non_zero_exit", False) and subp.wait(): raise EAException("Non-zero exit code while running command %s" % (' '.join(command))) except OSError as e: From 0a7e71bd658366d115e867b46f31209ba16ba3ca Mon Sep 17 00:00:00 2001 From: Caleb Collins-Parks <46505081+caleb15@users.noreply.github.com> Date: Fri, 6 Sep 2019 16:55:06 -0700 Subject: [PATCH 226/264] update jira to version that works with 3.7 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 412d53b1e..cb5e10eea 100644 --- a/setup.py +++ b/setup.py @@ -36,7 +36,7 @@ 'elasticsearch>=7.0.0', 'envparse>=0.2.0', 'exotel>=0.1.3', - 'jira>=1.0.10,<1.0.15', + 'jira>=2.0.0', 'jsonschema>=2.6.0,<3.0.0', 'mock>=2.0.0', 'PyStaticConfiguration>=0.10.3', From 16a464e208301e4e8e170bd6375b5cf65231346e Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Thu, 12 Sep 2019 11:34:04 -0400 Subject: [PATCH 227/264] Updating the docker test file to work with python 3.6 --- Dockerfile-test | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile-test b/Dockerfile-test index 761a777c6..3c153e644 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,9 +1,9 @@ FROM ubuntu:latest RUN apt-get update && apt-get upgrade -y -RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox python-pip +RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git WORKDIR /home/elastalert ADD requirements*.txt ./ -RUN pip install -r requirements-dev.txt +RUN pip3 install -r requirements-dev.txt From cfa98a12bb7739cfc21a16a418fe95375277f1fd Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 13 Sep 2019 13:49:25 -0400 Subject: [PATCH 228/264] Switching test_rule to use load_yaml --- elastalert/test_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 627dfe82c..ee467031b 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -410,7 +410,7 @@ def run_rule_test(self): args.es_debug_trace = False conf = load_conf(args, defaults, overwrites) - rule_yaml = conf['rules_loader'].get_yaml(args.file) + rule_yaml = conf['rules_loader'].load_yaml(args.file) conf['rules_loader'].load_options(rule_yaml, conf, args.file) if args.json: From 707b2a58a0599e1e21e9e0688314387046841250 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Wed, 18 Sep 2019 13:18:18 -0400 Subject: [PATCH 229/264] Handling non-compound query keys defined as an array --- elastalert/loaders.py | 12 +++++++++--- tests/loaders_test.py | 23 +++++++++++++++++++++++ 2 files changed, 32 insertions(+), 3 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index db14a3a52..ad05c6af2 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -324,9 +324,15 @@ def _dt_to_ts_with_format(dt): if 'include' in rule and type(rule['include']) != list: raise EAException('include option must be a list') - if isinstance(rule.get('query_key'), list): - rule['compound_query_key'] = rule['query_key'] - rule['query_key'] = ','.join(rule['query_key']) + raw_query_key = rule.get('query_key') + if isinstance(raw_query_key, list): + if len(raw_query_key) > 1: + rule['compound_query_key'] = raw_query_key + rule['query_key'] = ','.join(raw_query_key) + elif len(raw_query_key) == 1: + rule['query_key'] = raw_query_key[0] + else: + del(rule['query_key']) if isinstance(rule.get('aggregation_key'), list): rule['compound_aggregation_key'] = rule['aggregation_key'] diff --git a/tests/loaders_test.py b/tests/loaders_test.py index 509e8d4cb..40399d969 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -358,6 +358,29 @@ def test_compound_query_key(): assert test_rule_copy['compound_query_key'] == ['field1', 'field2'] +def test_query_key_with_single_value(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy.pop('use_count_query') + test_rule_copy['query_key'] = ['field1'] + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert 'field1' in test_rule_copy['include'] + assert test_rule_copy['query_key'] == 'field1' + assert 'compound_query_key' not in test_rule_copy + + +def test_query_key_with_no_values(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy.pop('use_count_query') + test_rule_copy['query_key'] = [] + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert 'query_key' not in test_rule_copy + assert 'compound_query_key' not in test_rule_copy + + def test_name_inference(): test_config_copy = copy.deepcopy(test_config) rules_loader = FileRulesLoader(test_config_copy) From 070bb1cd0ecad1d0de66ea57de408528d5fd4347 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Thu, 19 Sep 2019 14:00:12 -0400 Subject: [PATCH 230/264] Adding support for generating Kibana Discover app link --- docs/source/ruletypes.rst | 93 ++++ elastalert/elastalert.py | 6 + elastalert/kibana_discover.py | 192 ++++++++ elastalert/loaders.py | 4 + elastalert/schema.yaml | 9 + requirements.txt | 1 + setup.py | 1 + tests/kibana_discover_test.py | 858 ++++++++++++++++++++++++++++++++++ tests/loaders_test.py | 20 + 9 files changed, 1184 insertions(+) create mode 100644 elastalert/kibana_discover.py create mode 100644 tests/kibana_discover_test.py diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ba43a187f..6e606401a 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -58,6 +58,20 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana4_end_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | +| ``use_kibana_discover`` (boolean, default False) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_url`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_version`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_index_pattern_id`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_columns`` (list of strs, default _source) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_from_timedelta`` (time, default: 10 min) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_to_timedelta`` (time, default: 10 min) | | ++--------------------------------------------------------------+ | | ``use_local_time`` (boolean, default True) | | +--------------------------------------------------------------+ | | ``realert`` (time, default: 1 min) | | @@ -510,6 +524,85 @@ This value is added in back of the event. For example, ``kibana4_end_timedelta: minutes: 2`` +use_kibana_discover +^^^^^^^^^^^^^^^^^^^ + +``use_kibana_discover``: Enables the generation of the ``kibana_link`` variable for the Kibana Discover application. +This setting requires the following settings are also configured: + +- ``kibana_discover_url`` +- ``kibana_discover_version`` +- ``kibana_discover_index_pattern_id`` + +``use_kibana_discover: true`` + +kibana_discover_url +^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_url``: The url of the Kibana Discover application used to generate the ``kibana_link`` variable. +This value can use `$VAR` and `${VAR}` references to expand environment variables. + +``kibana_discover_url: http://kibana:5601/#/discover`` + +kibana_discover_version +^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_version``: Specifies the version of the Kibana Discover application. + +The currently supported versions of Kibana Discover are: + +- `5.6` +- `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` +- `7.0`, `7.1`, `7.2`, `7.3` + +``kibana_discover_version: '7.3'`` + +kibana_discover_index_pattern_id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_index_pattern_id``: The id of the index pattern to link to in the Kibana Discover application. +These ids are usually generated and can be found in url of the index pattern management page, or by exporting its saved object. + +Example export of an index pattern's saved object: + +.. code-block:: text + + [ + { + "_id": "4e97d188-8a45-4418-8a37-07ed69b4d34c", + "_type": "index-pattern", + "_source": { ... } + } + ] + +You can modify an index pattern's id by exporting the saved object, modifying the ``_id`` field, and re-importing. + +``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` + +kibana_discover_columns +^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link. +Defaults to the ``_source`` column. + +``kibana_discover_columns: [ timestamp, message ]`` + +kibana_discover_from_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_from_timedelta``: The offset to the `from` time of the Kibana Discover link's time range. +The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. + +``kibana_discover_from_timedelta: minutes: 2`` + +kibana_discover_to_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_to_timedelta``: The offset to the `to` time of the Kibana Discover link's time range. +The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. + +``kibana_discover_to_timedelta: minutes: 2`` + use_local_time ^^^^^^^^^^^^^^ diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 5436ada18..fdbdf076e 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -27,6 +27,7 @@ from elasticsearch.exceptions import NotFoundError from . import kibana +from .kibana_discover import kibana_discover_url from .alerts import DebugAlerter from .config import load_conf from .enhancements import DropMatchException @@ -1497,6 +1498,11 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): if kb_link: matches[0]['kibana_link'] = kb_link + if rule.get('use_kibana_discover'): + kb_link = kibana_discover_url(rule, matches[0]) + if kb_link: + matches[0]['kibana_link'] = kb_link + # Enhancements were already run at match time if # run_enhancements_first is set or # retried==True, which means this is a retry of a failed alert diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py new file mode 100644 index 000000000..edbc21b62 --- /dev/null +++ b/elastalert/kibana_discover.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import datetime +import logging +import json +import os.path +import prison +import urllib.parse + +from .util import EAException +from .util import lookup_es_key +from .util import ts_add + +kibana_default_timedelta = datetime.timedelta(minutes=10) + +kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) + +def kibana_discover_url(rule, match): + ''' Creates a link for a kibana discover app. ''' + + kibana_version = rule.get('kibana_discover_version') + if not kibana_version: + logging.warning( + 'use_kibana_discover was configured without kibana_discover_version for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + discover_url = rule.get('kibana_discover_url') + if not discover_url: + logging.warning( + 'use_kibana_discover was configured without kibana_discover_url for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + index = rule.get('kibana_discover_index_pattern_id') + if not index: + logging.warning( + 'use_kibana_discover was configured without kibana_discover_index_pattern_id for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + columns = rule.get('kibana_discover_columns', ['_source']) + filters = rule.get('filter', []) + + if 'query_key' in rule: + query_keys = rule.get('compound_query_key', [rule['query_key']]) + else: + query_keys = [] + + timestamp = lookup_es_key(match, rule['timestamp_field']) + timeframe = rule.get('timeframe', kibana_default_timedelta) + from_timedelta = rule.get('kibana_discover_from_timedelta', timeframe) + from_time = ts_add(timestamp, -from_timedelta) + to_timedelta = rule.get('kibana_discover_to_timedelta', timeframe) + to_time = ts_add(timestamp, to_timedelta) + + if kibana_version in kibana5_kibana6_versions: + globalState = kibana6_disover_global_state(from_time, to_time) + appState = kibana_discover_app_state(index, columns, filters, query_keys, match) + + elif kibana_version in kibana7_versions: + globalState = kibana7_disover_global_state(from_time, to_time) + appState = kibana_discover_app_state(index, columns, filters, query_keys, match) + + else: + logging.warning( + 'Unknown kibana discover application version %s for rule %s' % ( + kibana_version, + rule.get('name', '') + ) + ) + return None + + return "%s?_g=%s&_a=%s" % ( + os.path.expandvars(discover_url), + urllib.parse.quote(globalState), + urllib.parse.quote(appState) + ) + + +def kibana6_disover_global_state(from_time, to_time): + return prison.dumps( { + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'mode': 'absolute', + 'to': to_time + } + } ) + + +def kibana7_disover_global_state(from_time, to_time): + return prison.dumps( { + 'filters': [], + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'to': to_time + } + } ) + + +def kibana_discover_app_state(index, columns, filters, query_keys, match): + app_filters = [] + + if filters: + bool_filter = { 'must': filters } + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'bool': bool_filter, + 'meta': { + 'alias': 'filter', + 'disabled': False, + 'index': index, + 'key': 'bool', + 'negate': False, + 'type': 'custom', + 'value': json.dumps(bool_filter, separators=(',', ':')) + }, + } ) + + for query_key in query_keys: + query_value = lookup_es_key(match, query_key) + + if query_value is None: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'exists': { + 'field': query_key + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': True, + 'type': 'exists', + 'value': 'exists' + } + } ) + + else: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': False, + 'params': { + 'query': query_value, + 'type': 'phrase' + }, + 'type': 'phrase', + 'value': str(query_value) + }, + 'query': { + 'match': { + query_key: { + 'query': query_value, + 'type': 'phrase' + } + } + } + } ) + + return prison.dumps( { + 'columns': columns, + 'filters': app_filters, + 'index': index, + 'interval': 'auto' + } ) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index db14a3a52..806e22584 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -257,6 +257,10 @@ def load_options(self, rule, conf, filename, args=None): rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) if 'kibana4_end_timedelta' in rule: rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) + if 'kibana_discover_from_timedelta' in rule: + rule['kibana_discover_from_timedelta'] = datetime.timedelta(**rule['kibana_discover_from_timedelta']) + if 'kibana_discover_to_timedelta' in rule: + rule['kibana_discover_to_timedelta'] = datetime.timedelta(**rule['kibana_discover_to_timedelta']) except (KeyError, TypeError) as e: raise EAException('Invalid time format used: %s' % e) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 13562c332..953bed79f 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -203,6 +203,15 @@ properties: replace_dots_in_field_names: {type: boolean} scan_entire_timeframe: {type: boolean} + ### Kibana Discover App Link + use_kibana_discover: {type: boolean} + kibana_discover_url: {type: string} + kibana_discover_version: {enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_index_pattern_id: {type: string} + kibana_discover_columns: {type: array, items: {type: string}} + kibana_discover_from_timedelta: *timeframe + kibana_discover_to_timedelta: *timeframe + # Alert Content alert_text: {type: string} # Python format string alert_text_args: {type: array, items: {type: string}} diff --git a/requirements.txt b/requirements.txt index 3e5cbed90..0bd9521a8 100644 --- a/requirements.txt +++ b/requirements.txt @@ -12,6 +12,7 @@ exotel>=0.1.3 jira>=1.0.10,<1.0.15 jsonschema>=2.6.0 mock>=2.0.0 +prison>=0.1.2 py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 diff --git a/setup.py b/setup.py index 412d53b1e..26445bc20 100644 --- a/setup.py +++ b/setup.py @@ -39,6 +39,7 @@ 'jira>=1.0.10,<1.0.15', 'jsonschema>=2.6.0,<3.0.0', 'mock>=2.0.0', + 'prison>=0.1.2', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', 'PyYAML>=3.12', diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py new file mode 100644 index 000000000..cb3f97630 --- /dev/null +++ b/tests/kibana_discover_test.py @@ -0,0 +1,858 @@ +# -*- coding: utf-8 -*- +from datetime import timedelta +import pytest + +from elastalert.kibana_discover import kibana_discover_url + + +@pytest.mark.parametrize("kibana_version", ['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) +def test_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': kibana_version, + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) +def test_kibana_discover_url_with_kibana_7x(kibana_version): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': kibana_version, + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_missing_kibana_discover_version(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_kibana_discover_url_with_missing_kibana_discover_url(): + url = kibana_discover_url( + rule={ + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_kibana_discover_url_with_invalid_kibana_version(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '4.5', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_kibana_discover_url_with_discover_url_env_substitution(environ): + environ.update({ + 'KIBANA_HOST': 'kibana', + 'KIBANA_PORT': '5601', + }) + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_from_timedelta(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_from_timedelta': timedelta(hours=1), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A10%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_from_timedelta_and_timeframe(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_from_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A20%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_to_timedelta(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_to_timedelta': timedelta(hours=1), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A50%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_to_timedelta_and_timeframe(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_to_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A40%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_timeframe(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T04%3A10%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A50%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_custom_columns(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'kibana_discover_columns': ['level', 'message'], + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28level%2Cmessage%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_single_filter(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_multiple_filters(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'app': 'test'}}, + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B' # value start + + '%7B%22term%22%3A%7B%22app%22%3A%22test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D' + + '%5D%7D%27' # value end + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_int_query_key(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo.dest': 200 + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3A200%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3A%27200%27' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # reponse start + + 'query%3A200%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_str_query_key(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'dest': 'ok' + } + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3Aok%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3Aok' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # geo.dest start + + 'query%3Aok%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_null_query_key_value(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': None + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'exists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21t%2C' + + 'type%3Aexists%2C' + + 'value%3Aexists' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_missing_query_key_value(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'exists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21t%2C' + + 'type%3Aexists%2C' + + 'value%3Aexists' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_compound_query_key(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'compound_query_key': ['geo.src', 'geo.dest'], + 'query_key': 'geo.src,geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'src': 'CA', + 'dest': 'US' + } + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # geo.src filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.src%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3ACA%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3ACA' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.src%3A%28' # reponse start + + 'query%3ACA%2C' + + 'type%3Aphrase' + + '%29' # geo.src end + + '%29' # match end + + '%29' # query end + + '%29%2C' # geo.src filter end + + + '%28' # geo.dest filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3AUS%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3AUS' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # geo.dest start + + 'query%3AUS%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # geo.dest filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_kibana_discover_url_with_filter_and_query_key(): + url = kibana_discover_url( + rule={ + 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ], + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': 'ok' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27' + + '%29' # meta end + + '%29%2C' # filter end + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3Aok%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3Aok' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'status%3A%28' # status start + + 'query%3Aok%2C' + + 'type%3Aphrase' + + '%29' # status end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl diff --git a/tests/loaders_test.py b/tests/loaders_test.py index 509e8d4cb..38793878c 100644 --- a/tests/loaders_test.py +++ b/tests/loaders_test.py @@ -395,3 +395,23 @@ def test_raises_on_bad_generate_kibana_filters(): test_rule_copy['filter'] = good + bad with pytest.raises(EAException): rules_loader.load_configuration('blah', test_config) + + +def test_kibana_discover_from_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['kibana_discover_from_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['kibana_discover_from_timedelta'], datetime.timedelta) + assert test_rule_copy['kibana_discover_from_timedelta'] == datetime.timedelta(minutes=2) + + +def test_kibana_discover_to_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['kibana_discover_to_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['kibana_discover_to_timedelta'], datetime.timedelta) + assert test_rule_copy['kibana_discover_to_timedelta'] == datetime.timedelta(minutes=2) From 89b96ade4ace42ed37427404613372e759fadae7 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 11:27:21 -0400 Subject: [PATCH 231/264] Renaming the kibana_discover_link function --- elastalert/elastalert.py | 4 +- elastalert/kibana_discover.py | 2 +- tests/kibana_discover_test.py | 86 +++++++++++++++++------------------ 3 files changed, 46 insertions(+), 46 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index fdbdf076e..c006b2ae3 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -27,7 +27,7 @@ from elasticsearch.exceptions import NotFoundError from . import kibana -from .kibana_discover import kibana_discover_url +from .kibana_discover import generate_kibana_discover_link from .alerts import DebugAlerter from .config import load_conf from .enhancements import DropMatchException @@ -1499,7 +1499,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): matches[0]['kibana_link'] = kb_link if rule.get('use_kibana_discover'): - kb_link = kibana_discover_url(rule, matches[0]) + kb_link = generate_kibana_discover_link(rule, matches[0]) if kb_link: matches[0]['kibana_link'] = kb_link diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index edbc21b62..1c5a27cc7 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -16,7 +16,7 @@ kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) -def kibana_discover_url(rule, match): +def generate_kibana_discover_link(rule, match): ''' Creates a link for a kibana discover app. ''' kibana_version = rule.get('kibana_discover_version') diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index cb3f97630..8e5060d80 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -2,12 +2,12 @@ from datetime import timedelta import pytest -from elastalert.kibana_discover import kibana_discover_url +from elastalert.kibana_discover import generate_kibana_discover_link @pytest.mark.parametrize("kibana_version", ['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -def test_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_kibana_5x_and_6x(kibana_version): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': kibana_version, @@ -39,8 +39,8 @@ def test_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): @pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) -def test_kibana_discover_url_with_kibana_7x(kibana_version): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_kibana_7x(kibana_version): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': kibana_version, @@ -71,8 +71,8 @@ def test_kibana_discover_url_with_kibana_7x(kibana_version): assert url == expectedUrl -def test_kibana_discover_url_with_missing_kibana_discover_version(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_missing_kibana_discover_version(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_index_pattern_id': 'logs', @@ -86,8 +86,8 @@ def test_kibana_discover_url_with_missing_kibana_discover_version(): assert url is None -def test_kibana_discover_url_with_missing_kibana_discover_url(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_missing_kibana_discover_url(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs', @@ -101,8 +101,8 @@ def test_kibana_discover_url_with_missing_kibana_discover_url(): assert url is None -def test_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_missing_kibana_discover_index_pattern_id(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -116,8 +116,8 @@ def test_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): assert url is None -def test_kibana_discover_url_with_invalid_kibana_version(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_invalid_kibana_version(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '4.5', @@ -131,12 +131,12 @@ def test_kibana_discover_url_with_invalid_kibana_version(): assert url is None -def test_kibana_discover_url_with_discover_url_env_substitution(environ): +def test_generate_kibana_discover_link_with_discover_url_env_substitution(environ): environ.update({ 'KIBANA_HOST': 'kibana', 'KIBANA_PORT': '5601', }) - url = kibana_discover_url( + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', 'kibana_discover_version': '6.8', @@ -167,8 +167,8 @@ def test_kibana_discover_url_with_discover_url_env_substitution(environ): assert url == expectedUrl -def test_kibana_discover_url_with_from_timedelta(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_from_timedelta(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -200,8 +200,8 @@ def test_kibana_discover_url_with_from_timedelta(): assert url == expectedUrl -def test_kibana_discover_url_with_from_timedelta_and_timeframe(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_from_timedelta_and_timeframe(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -234,8 +234,8 @@ def test_kibana_discover_url_with_from_timedelta_and_timeframe(): assert url == expectedUrl -def test_kibana_discover_url_with_to_timedelta(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_to_timedelta(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -267,8 +267,8 @@ def test_kibana_discover_url_with_to_timedelta(): assert url == expectedUrl -def test_kibana_discover_url_with_to_timedelta_and_timeframe(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_to_timedelta_and_timeframe(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -301,8 +301,8 @@ def test_kibana_discover_url_with_to_timedelta_and_timeframe(): assert url == expectedUrl -def test_kibana_discover_url_with_timeframe(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_timeframe(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', @@ -334,8 +334,8 @@ def test_kibana_discover_url_with_timeframe(): assert url == expectedUrl -def test_kibana_discover_url_with_custom_columns(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_custom_columns(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -367,8 +367,8 @@ def test_kibana_discover_url_with_custom_columns(): assert url == expectedUrl -def test_kibana_discover_url_with_single_filter(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_single_filter(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -418,8 +418,8 @@ def test_kibana_discover_url_with_single_filter(): assert url == expectedUrl -def test_kibana_discover_url_with_multiple_filters(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_multiple_filters(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -472,8 +472,8 @@ def test_kibana_discover_url_with_multiple_filters(): assert url == expectedUrl -def test_kibana_discover_url_with_int_query_key(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_int_query_key(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -532,8 +532,8 @@ def test_kibana_discover_url_with_int_query_key(): assert url == expectedUrl -def test_kibana_discover_url_with_str_query_key(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_str_query_key(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -594,8 +594,8 @@ def test_kibana_discover_url_with_str_query_key(): assert url == expectedUrl -def test_kibana_discover_url_with_null_query_key_value(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_null_query_key_value(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -644,8 +644,8 @@ def test_kibana_discover_url_with_null_query_key_value(): assert url == expectedUrl -def test_kibana_discover_url_with_missing_query_key_value(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_missing_query_key_value(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -693,8 +693,8 @@ def test_kibana_discover_url_with_missing_query_key_value(): assert url == expectedUrl -def test_kibana_discover_url_with_compound_query_key(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_compound_query_key(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', @@ -781,8 +781,8 @@ def test_kibana_discover_url_with_compound_query_key(): assert url == expectedUrl -def test_kibana_discover_url_with_filter_and_query_key(): - url = kibana_discover_url( +def test_generate_kibana_discover_link_with_filter_and_query_key(): + url = generate_kibana_discover_link( rule={ 'kibana_discover_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', From 47cad175dce5966edadbe3abd553b667325d2543 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 11:35:08 -0400 Subject: [PATCH 232/264] Pruning the duplicate requirements --- requirements.txt | 2 -- 1 file changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e5cbed90..d5fbaa46a 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,7 +3,6 @@ aws-requests-auth>=0.3.0 blist>=1.3.6 boto3>=1.4.4 cffi>=1.11.5 -cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 elasticsearch>=7.0.0 @@ -16,7 +15,6 @@ py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 python-magic>=0.4.15 -python-magic>=0.4.15 PyYAML>=5.1 requests>=2.0.0 stomp.py>=4.1.17 From e5d20eaa0c79e9053c320a3211ed41f9271855b5 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 12:45:59 -0400 Subject: [PATCH 233/264] Generate kibana discover link (#3) * Renaming config to generate-kibana-discover-link --- docs/source/ruletypes.rst | 12 ++++++------ elastalert/elastalert.py | 4 ++-- elastalert/kibana_discover.py | 14 +++++++------- elastalert/schema.yaml | 2 +- 4 files changed, 16 insertions(+), 16 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 6e606401a..d550a8c93 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -58,7 +58,7 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana4_end_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | -| ``use_kibana_discover`` (boolean, default False) | | +| ``generate_kibana_discover_link`` (boolean, default False) | | +--------------------------------------------------------------+ | | ``kibana_discover_url`` (string, no default) | | +--------------------------------------------------------------+ | @@ -524,22 +524,22 @@ This value is added in back of the event. For example, ``kibana4_end_timedelta: minutes: 2`` -use_kibana_discover -^^^^^^^^^^^^^^^^^^^ +generate_kibana_discover_link +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``use_kibana_discover``: Enables the generation of the ``kibana_link`` variable for the Kibana Discover application. +``generate_kibana_discover_link``: Enables the generation of the ``kibana_discover_link`` variable for the Kibana Discover application. This setting requires the following settings are also configured: - ``kibana_discover_url`` - ``kibana_discover_version`` - ``kibana_discover_index_pattern_id`` -``use_kibana_discover: true`` +``generate_kibana_discover_link: true`` kibana_discover_url ^^^^^^^^^^^^^^^^^^^^ -``kibana_discover_url``: The url of the Kibana Discover application used to generate the ``kibana_link`` variable. +``kibana_discover_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_link`` variable. This value can use `$VAR` and `${VAR}` references to expand environment variables. ``kibana_discover_url: http://kibana:5601/#/discover`` diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index c006b2ae3..5cc82b554 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -1498,10 +1498,10 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): if kb_link: matches[0]['kibana_link'] = kb_link - if rule.get('use_kibana_discover'): + if rule.get('generate_kibana_discover_link'): kb_link = generate_kibana_discover_link(rule, matches[0]) if kb_link: - matches[0]['kibana_link'] = kb_link + matches[0]['kibana_discover_link'] = kb_link # Enhancements were already run at match time if # run_enhancements_first is set or diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 1c5a27cc7..729e0ea29 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -19,19 +19,19 @@ def generate_kibana_discover_link(rule, match): ''' Creates a link for a kibana discover app. ''' - kibana_version = rule.get('kibana_discover_version') - if not kibana_version: + discover_url = rule.get('kibana_discover_url') + if not discover_url: logging.warning( - 'use_kibana_discover was configured without kibana_discover_version for rule %s' % ( + 'Missing kibana_discover_url for rule %s' % ( rule.get('name', '') ) ) return None - discover_url = rule.get('kibana_discover_url') - if not discover_url: + kibana_version = rule.get('kibana_discover_version') + if not kibana_version: logging.warning( - 'use_kibana_discover was configured without kibana_discover_url for rule %s' % ( + 'Missing kibana_discover_version for rule %s' % ( rule.get('name', '') ) ) @@ -40,7 +40,7 @@ def generate_kibana_discover_link(rule, match): index = rule.get('kibana_discover_index_pattern_id') if not index: logging.warning( - 'use_kibana_discover was configured without kibana_discover_index_pattern_id for rule %s' % ( + 'Missing kibana_discover_index_pattern_id for rule %s' % ( rule.get('name', '') ) ) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 953bed79f..8a7294a6f 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -204,7 +204,7 @@ properties: scan_entire_timeframe: {type: boolean} ### Kibana Discover App Link - use_kibana_discover: {type: boolean} + generate_kibana_discover_link: {type: boolean} kibana_discover_url: {type: string} kibana_discover_version: {enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string} From 82760140b83672f6dced41bf28053fdcb1fa31cf Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 13:53:59 -0400 Subject: [PATCH 234/264] Renaming to generate_kibana_discover_url (#4) --- docs/source/ruletypes.rst | 32 ++++----- elastalert/elastalert.py | 8 +-- elastalert/kibana_discover.py | 10 +-- elastalert/schema.yaml | 4 +- tests/kibana_discover_test.py | 126 +++++++++++++++++----------------- 5 files changed, 90 insertions(+), 90 deletions(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index d550a8c93..9b827b753 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -58,9 +58,9 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana4_end_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | -| ``generate_kibana_discover_link`` (boolean, default False) | | +| ``generate_kibana_discover_url`` (boolean, default False) | | +--------------------------------------------------------------+ | -| ``kibana_discover_url`` (string, no default) | | +| ``kibana_discover_app_url`` (string, no default) | | +--------------------------------------------------------------+ | | ``kibana_discover_version`` (string, no default) | | +--------------------------------------------------------------+ | @@ -524,28 +524,28 @@ This value is added in back of the event. For example, ``kibana4_end_timedelta: minutes: 2`` -generate_kibana_discover_link -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +generate_kibana_discover_url +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -``generate_kibana_discover_link``: Enables the generation of the ``kibana_discover_link`` variable for the Kibana Discover application. +``generate_kibana_discover_url``: Enables the generation of the ``kibana_discover_url`` variable for the Kibana Discover application. This setting requires the following settings are also configured: -- ``kibana_discover_url`` +- ``kibana_discover_app_url`` - ``kibana_discover_version`` - ``kibana_discover_index_pattern_id`` -``generate_kibana_discover_link: true`` +``generate_kibana_discover_url: true`` -kibana_discover_url -^^^^^^^^^^^^^^^^^^^^ +kibana_discover_app_url +^^^^^^^^^^^^^^^^^^^^^^^ -``kibana_discover_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_link`` variable. +``kibana_discover_app_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_url`` variable. This value can use `$VAR` and `${VAR}` references to expand environment variables. -``kibana_discover_url: http://kibana:5601/#/discover`` +``kibana_discover_app_url: http://kibana:5601/#/discover`` kibana_discover_version -^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_version``: Specifies the version of the Kibana Discover application. @@ -558,7 +558,7 @@ The currently supported versions of Kibana Discover are: ``kibana_discover_version: '7.3'`` kibana_discover_index_pattern_id -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_index_pattern_id``: The id of the index pattern to link to in the Kibana Discover application. These ids are usually generated and can be found in url of the index pattern management page, or by exporting its saved object. @@ -580,7 +580,7 @@ You can modify an index pattern's id by exporting the saved object, modifying th ``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` kibana_discover_columns -^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link. Defaults to the ``_source`` column. @@ -588,7 +588,7 @@ Defaults to the ``_source`` column. ``kibana_discover_columns: [ timestamp, message ]`` kibana_discover_from_timedelta -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_from_timedelta``: The offset to the `from` time of the Kibana Discover link's time range. The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. @@ -596,7 +596,7 @@ The `from` time is calculated by subtracting this timedelta from the event time. ``kibana_discover_from_timedelta: minutes: 2`` kibana_discover_to_timedelta -^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ ``kibana_discover_to_timedelta``: The offset to the `to` time of the Kibana Discover link's time range. The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 5cc82b554..01beb57d8 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -27,7 +27,7 @@ from elasticsearch.exceptions import NotFoundError from . import kibana -from .kibana_discover import generate_kibana_discover_link +from .kibana_discover import generate_kibana_discover_url from .alerts import DebugAlerter from .config import load_conf from .enhancements import DropMatchException @@ -1498,10 +1498,10 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): if kb_link: matches[0]['kibana_link'] = kb_link - if rule.get('generate_kibana_discover_link'): - kb_link = generate_kibana_discover_link(rule, matches[0]) + if rule.get('generate_kibana_discover_url'): + kb_link = generate_kibana_discover_url(rule, matches[0]) if kb_link: - matches[0]['kibana_discover_link'] = kb_link + matches[0]['kibana_discover_url'] = kb_link # Enhancements were already run at match time if # run_enhancements_first is set or diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py index 729e0ea29..7e4dbb5d1 100644 --- a/elastalert/kibana_discover.py +++ b/elastalert/kibana_discover.py @@ -16,13 +16,13 @@ kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) -def generate_kibana_discover_link(rule, match): +def generate_kibana_discover_url(rule, match): ''' Creates a link for a kibana discover app. ''' - discover_url = rule.get('kibana_discover_url') - if not discover_url: + discover_app_url = rule.get('kibana_discover_app_url') + if not discover_app_url: logging.warning( - 'Missing kibana_discover_url for rule %s' % ( + 'Missing kibana_discover_app_url for rule %s' % ( rule.get('name', '') ) ) @@ -79,7 +79,7 @@ def generate_kibana_discover_link(rule, match): return None return "%s?_g=%s&_a=%s" % ( - os.path.expandvars(discover_url), + os.path.expandvars(discover_app_url), urllib.parse.quote(globalState), urllib.parse.quote(appState) ) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 8a7294a6f..2190c05f1 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -204,8 +204,8 @@ properties: scan_entire_timeframe: {type: boolean} ### Kibana Discover App Link - generate_kibana_discover_link: {type: boolean} - kibana_discover_url: {type: string} + generate_kibana_discover_url: {type: boolean} + kibana_discover_app_url: {type: string} kibana_discover_version: {enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string} kibana_discover_columns: {type: array, items: {type: string}} diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py index 8e5060d80..f06fe4e0c 100644 --- a/tests/kibana_discover_test.py +++ b/tests/kibana_discover_test.py @@ -2,14 +2,14 @@ from datetime import timedelta import pytest -from elastalert.kibana_discover import generate_kibana_discover_link +from elastalert.kibana_discover import generate_kibana_discover_url @pytest.mark.parametrize("kibana_version", ['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) -def test_generate_kibana_discover_link_with_kibana_5x_and_6x(kibana_version): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': kibana_version, 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timestamp_field': 'timestamp' @@ -39,10 +39,10 @@ def test_generate_kibana_discover_link_with_kibana_5x_and_6x(kibana_version): @pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) -def test_generate_kibana_discover_link_with_kibana_7x(kibana_version): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': kibana_version, 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timestamp_field': 'timestamp' @@ -71,10 +71,10 @@ def test_generate_kibana_discover_link_with_kibana_7x(kibana_version): assert url == expectedUrl -def test_generate_kibana_discover_link_with_missing_kibana_discover_version(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_missing_kibana_discover_version(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_index_pattern_id': 'logs', 'timestamp_field': 'timestamp', 'name': 'test' @@ -86,8 +86,8 @@ def test_generate_kibana_discover_link_with_missing_kibana_discover_version(): assert url is None -def test_generate_kibana_discover_link_with_missing_kibana_discover_url(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_missing_kibana_discover_app_url(): + url = generate_kibana_discover_url( rule={ 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs', @@ -101,10 +101,10 @@ def test_generate_kibana_discover_link_with_missing_kibana_discover_url(): assert url is None -def test_generate_kibana_discover_link_with_missing_kibana_discover_index_pattern_id(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'timestamp_field': 'timestamp', 'name': 'test' @@ -116,10 +116,10 @@ def test_generate_kibana_discover_link_with_missing_kibana_discover_index_patter assert url is None -def test_generate_kibana_discover_link_with_invalid_kibana_version(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_invalid_kibana_version(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '4.5', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp' @@ -131,14 +131,14 @@ def test_generate_kibana_discover_link_with_invalid_kibana_version(): assert url is None -def test_generate_kibana_discover_link_with_discover_url_env_substitution(environ): +def test_generate_kibana_discover_url_with_kibana_discover_app_url_env_substitution(environ): environ.update({ 'KIBANA_HOST': 'kibana', 'KIBANA_PORT': '5601', }) - url = generate_kibana_discover_link( + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', + 'kibana_discover_app_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timestamp_field': 'timestamp' @@ -167,10 +167,10 @@ def test_generate_kibana_discover_link_with_discover_url_env_substitution(enviro assert url == expectedUrl -def test_generate_kibana_discover_link_with_from_timedelta(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_from_timedelta(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), @@ -200,10 +200,10 @@ def test_generate_kibana_discover_link_with_from_timedelta(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_from_timedelta_and_timeframe(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_from_timedelta': timedelta(hours=1), @@ -234,10 +234,10 @@ def test_generate_kibana_discover_link_with_from_timedelta_and_timeframe(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_to_timedelta(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_to_timedelta(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), @@ -267,10 +267,10 @@ def test_generate_kibana_discover_link_with_to_timedelta(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_to_timedelta_and_timeframe(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'kibana_discover_to_timedelta': timedelta(hours=1), @@ -301,10 +301,10 @@ def test_generate_kibana_discover_link_with_to_timedelta_and_timeframe(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_timeframe(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_timeframe(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '7.3', 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', 'timeframe': timedelta(minutes=20), @@ -334,10 +334,10 @@ def test_generate_kibana_discover_link_with_timeframe(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_custom_columns(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_custom_columns(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'kibana_discover_columns': ['level', 'message'], @@ -367,10 +367,10 @@ def test_generate_kibana_discover_link_with_custom_columns(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_single_filter(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_single_filter(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -418,10 +418,10 @@ def test_generate_kibana_discover_link_with_single_filter(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_multiple_filters(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_multiple_filters(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', 'timestamp_field': 'timestamp', @@ -472,10 +472,10 @@ def test_generate_kibana_discover_link_with_multiple_filters(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_int_query_key(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_int_query_key(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -532,10 +532,10 @@ def test_generate_kibana_discover_link_with_int_query_key(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_str_query_key(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_str_query_key(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -594,10 +594,10 @@ def test_generate_kibana_discover_link_with_str_query_key(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_null_query_key_value(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_null_query_key_value(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -644,10 +644,10 @@ def test_generate_kibana_discover_link_with_null_query_key_value(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_missing_query_key_value(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_missing_query_key_value(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -693,10 +693,10 @@ def test_generate_kibana_discover_link_with_missing_query_key_value(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_compound_query_key(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_compound_query_key(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', @@ -781,10 +781,10 @@ def test_generate_kibana_discover_link_with_compound_query_key(): assert url == expectedUrl -def test_generate_kibana_discover_link_with_filter_and_query_key(): - url = generate_kibana_discover_link( +def test_generate_kibana_discover_url_with_filter_and_query_key(): + url = generate_kibana_discover_url( rule={ - 'kibana_discover_url': 'http://kibana:5601/#/discover', + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', 'kibana_discover_version': '6.8', 'kibana_discover_index_pattern_id': 'logs-*', 'timestamp_field': 'timestamp', From 53ab1ba18d9a40c6aa820fda1c7b2ea83218a126 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 14:48:01 -0400 Subject: [PATCH 235/264] Ensuring the version is a string (#5) --- elastalert/schema.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 2190c05f1..62c2456b1 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -206,7 +206,7 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} kibana_discover_app_url: {type: string} - kibana_discover_version: {enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_version: {type: string, enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} kibana_discover_index_pattern_id: {type: string} kibana_discover_columns: {type: array, items: {type: string}} kibana_discover_from_timedelta: *timeframe From d719203ae0a10bea460b19884a5650bb7335d2ac Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 15:26:03 -0400 Subject: [PATCH 236/264] Upgrading jsonschema library to 3.0.2 --- requirements.txt | 2 +- setup.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index 3e5cbed90..39276ee3b 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ elasticsearch>=7.0.0 envparse>=0.2.0 exotel>=0.1.3 jira>=1.0.10,<1.0.15 -jsonschema>=2.6.0 +jsonschema>=3.0.2 mock>=2.0.0 py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 diff --git a/setup.py b/setup.py index 412d53b1e..341b9fc7a 100644 --- a/setup.py +++ b/setup.py @@ -37,7 +37,7 @@ 'envparse>=0.2.0', 'exotel>=0.1.3', 'jira>=1.0.10,<1.0.15', - 'jsonschema>=2.6.0,<3.0.0', + 'jsonschema>=3.0.2', 'mock>=2.0.0', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', From 28e3ff15dc9dbcc6e55c747e72f6b2f69147965f Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 15:51:45 -0400 Subject: [PATCH 237/264] Updating to use draft 7 --- elastalert/loaders.py | 2 +- elastalert/schema.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index db14a3a52..0f28f0b69 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -90,7 +90,7 @@ class RulesLoader(object): def __init__(self, conf): # schema for rule yaml - self.rule_schema = jsonschema.Draft4Validator( + self.rule_schema = jsonschema.Draft7Validator( yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader)) self.base_config = copy.deepcopy(conf) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 13562c332..0135bdf55 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -1,4 +1,4 @@ -$schema: http://json-schema.org/draft-04/schema# +$schema: http://json-schema.org/draft-07/schema# definitions: # Either a single string OR an array of strings From 29b380e1497dbc3407e3189a02e1c1933fdb8eb9 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 16:05:42 -0400 Subject: [PATCH 238/264] Fixing the outstanding documentation warnings --- docs/source/recipes/writing_filters.rst | 2 +- docs/source/ruletypes.rst | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/source/recipes/writing_filters.rst b/docs/source/recipes/writing_filters.rst index e923b89a5..1d2959262 100644 --- a/docs/source/recipes/writing_filters.rst +++ b/docs/source/recipes/writing_filters.rst @@ -57,7 +57,7 @@ a field that appears to have the value "foo bar", unless it is not analyzed. Con matching on analyzed fields, use query_string. See https://www.elastic.co/guide/en/elasticsearch/guide/current/term-vs-full-text.html `terms `_ -***** +***************************************************************************************************** diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ba43a187f..f7dc65337 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1305,7 +1305,7 @@ With ``alert_text_type: aggregation_summary_only``:: body = rule_name aggregation_summary -+ + ruletype_text is the string returned by RuleType.get_match_str. field_values will contain every key value pair included in the results from Elasticsearch. These fields include "@timestamp" (or the value of ``timestamp_field``), @@ -1689,7 +1689,7 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. Mattermost -~~~~~ +~~~~~~~~~~ Mattermost alerter will send a notification to a predefined Mattermost channel. The body of the notification is formatted the same as with other alerters. From 9178b7d02d98fdecba259fe7f2172e75976b714e Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 16:11:38 -0400 Subject: [PATCH 239/264] Removing empty header --- docs/source/elastalert.rst | 1 - 1 file changed, 1 deletion(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 131d1742e..970cde7aa 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -203,7 +203,6 @@ The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticse ``skip_invalid``: If ``True``, skip invalid files instead of exiting. -======= Logging ------- From 8ddfa7032d0bc580f416ef5b97e30bfab89afdb1 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 16:32:24 -0400 Subject: [PATCH 240/264] Treating documentation warnings as errors --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 6a6efc293..71099e17c 100644 --- a/tox.ini +++ b/tox.ini @@ -27,4 +27,4 @@ norecursedirs = .* virtualenv_run docs build venv env deps = {[testenv]deps} sphinx==1.6.6 changedir = docs -commands = sphinx-build -b html -d build/doctrees source build/html +commands = sphinx-build -b html -d build/doctrees -W source build/html From 1392521d59fcfaafefce161b83fb64eb72218f9e Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Fri, 20 Sep 2019 17:59:55 -0400 Subject: [PATCH 241/264] Making schema more strict (#8) * Making schema more strict --- elastalert/schema.yaml | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 62c2456b1..750563eee 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -11,6 +11,17 @@ definitions: type: [string, array] items: {type: [string, array]} + timedelta: &timedelta + type: object + additionalProperties: false + properties: + days: {type: number} + weeks: {type: number} + hours: {type: number} + minutes: {type: number} + seconds: {type: number} + milliseconds: {type: number} + timeFrame: &timeframe type: object additionalProperties: false @@ -205,12 +216,12 @@ properties: ### Kibana Discover App Link generate_kibana_discover_url: {type: boolean} - kibana_discover_app_url: {type: string} + kibana_discover_app_url: {type: string, format: uri} kibana_discover_version: {type: string, enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} - kibana_discover_index_pattern_id: {type: string} - kibana_discover_columns: {type: array, items: {type: string}} - kibana_discover_from_timedelta: *timeframe - kibana_discover_to_timedelta: *timeframe + kibana_discover_index_pattern_id: {type: string, minLength: 1} + kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} + kibana_discover_from_timedelta: *timedelta + kibana_discover_to_timedelta: *timedelta # Alert Content alert_text: {type: string} # Python format string From db10b2ca4f1cfbc1302f1e843c13b15e4f8b845a Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 23 Sep 2019 14:55:35 -0700 Subject: [PATCH 242/264] Stop job when it gets disabled --- elastalert/elastalert.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 5436ada18..e1c0dae92 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -23,8 +23,8 @@ from croniter import croniter from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException -from elasticsearch.exceptions import TransportError from elasticsearch.exceptions import NotFoundError +from elasticsearch.exceptions import TransportError from . import kibana from .alerts import DebugAlerter @@ -1067,7 +1067,13 @@ def load_rule_changes(self): if rule_file not in new_rule_hashes: # Rule file was deleted elastalert_logger.info('Rule file %s not found, stopping rule execution' % (rule_file)) - self.rules = [rule for rule in self.rules if rule['rule_file'] != rule_file] + for rule in self.rules: + if rule['rule_file'] == rule_file: + break + else: + continue + self.scheduler.remove_job(job_id=rule['name']) + self.rules.remove(rule) continue if hash_value != new_rule_hashes[rule_file]: # Rule file was changed, reload rule From 3c0aa03945031cf55e7459bfa5631fcb46c3219b Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Mon, 23 Sep 2019 15:25:53 -0700 Subject: [PATCH 243/264] Added a test for rule scheduler removal --- tests/base_test.py | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/tests/base_test.py b/tests/base_test.py index 15474c690..0e57a2ff9 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1049,6 +1049,15 @@ def test_rule_changes(ea): ea.load_rule_changes() assert len(ea.rules) == 4 + # Disable a rule by removing the file + new_hashes.pop('rules/rule4.yaml') + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} + mock_hashes.return_value = new_hashes + ea.load_rule_changes() + ea.scheduler.remove_job.assert_called_with(job_id='rule4') + def test_strf_index(ea): """ Test that the get_index function properly generates indexes spanning days """ From a02b146dba509d852664d556e6e18435abf51dd9 Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Wed, 25 Sep 2019 14:46:35 -0400 Subject: [PATCH 244/264] Adding ability to attach the Kibana Discover url as a seperate attachment in Slack notifications --- docs/source/ruletypes.rst | 6 ++ elastalert/alerts.py | 12 +++ elastalert/schema.yaml | 3 + tests/alerts_test.py | 200 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 221 insertions(+) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 49e2e5dea..3e0f78b9b 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1781,6 +1781,12 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. +``slack_attach_kibana_discover_url``: Enables the attachment of the ``kibana_discover_url`` to the slack notification. The config ``generate_kibana_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. + +``slack_kibana_discover_color``: The color of the Kibana Discover url attachment. Defaults to ``#ec4b98``. + +``slack_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. + Mattermost ~~~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 9efa765c5..f5ca22070 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -1129,6 +1129,9 @@ def __init__(self, rule): self.slack_ignore_ssl_errors = self.rule.get('slack_ignore_ssl_errors', False) self.slack_timeout = self.rule.get('slack_timeout', 10) self.slack_ca_certs = self.rule.get('slack_ca_certs') + self.slack_attach_kibana_discover_url = self.rule.get('slack_attach_kibana_discover_url', False) + self.slack_kibana_discover_color = self.rule.get('slack_kibana_discover_color', '#ec4b98') + self.slack_kibana_discover_title = self.rule.get('slack_kibana_discover_title', 'Discover in Kibana') def format_body(self, body): # https://api.slack.com/docs/formatting @@ -1191,6 +1194,15 @@ def alert(self, matches): if self.slack_title_link != '': payload['attachments'][0]['title_link'] = self.slack_title_link + if self.slack_attach_kibana_discover_url: + kibana_discover_url = lookup_es_key(matches[0], 'kibana_discover_url') + if kibana_discover_url: + payload['attachments'].append({ + 'color': self.slack_kibana_discover_color, + 'title': self.slack_kibana_discover_title, + 'title_link': kibana_discover_url + }) + for url in self.slack_webhook_url: for channel_override in self.slack_channel_override: try: diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 6cdad648e..afdffa20b 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -286,6 +286,9 @@ properties: slack_text_string: {type: string} slack_ignore_ssl_errors: {type: boolean} slack_ca_certs: {type: string} + slack_attach_kibana_discover_url {type: boolean} + slack_kibana_discover_color {type: string} + slack_kibana_discover_title {type: string} ### Mattermost mattermost_webhook_url: *arrayOfString diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 549b60624..da21f1b38 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1190,6 +1190,206 @@ def test_slack_uses_list_of_custom_slack_channel(): assert expected_data2 == json.loads(mock_post_request.call_args_list[1][1]['data']) +def test_slack_attach_kibana_discover_url_when_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_attach_kibana_discover_url_when_not_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_kibana_discover_title(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_kibana_discover_title': 'Click to discover in Kibana', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Click to discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_kibana_discover_color(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_kibana_discover_color': 'blue', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': 'blue', + 'title': 'Discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + def test_http_alerter_with_payload(): rule = { 'name': 'Test HTTP Post Alerter With Payload', From 74ef682783b0d5ed185395f1198f3d4a7d37b0cb Mon Sep 17 00:00:00 2001 From: Jeff Ashton Date: Thu, 26 Sep 2019 14:10:37 -0400 Subject: [PATCH 245/264] Adding ability to map match fields into opsgenie details --- docs/source/ruletypes.rst | 9 ++ elastalert/opsgenie.py | 23 +++- elastalert/schema.yaml | 14 +++ tests/alerts_test.py | 255 ++++++++++++++++++++++++++++++++++++++ 4 files changed, 300 insertions(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 49e2e5dea..e9454c259 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1648,6 +1648,15 @@ Optional: ``opsgenie_priority``: Set the OpsGenie priority level. Possible values are P1, P2, P3, P4, P5. +``opsgenie_details``: Map of custom key/value pairs to include in the alert's details. The value can sourced from either fields in the first match, environment variables, or a constant value. + +Example usage:: + + opsgenie_details: + Author: 'Bob Smith' # constant value + Environment: '$VAR' # environment variable + Message: { field: message } # field in the first match + SNS ~~~ diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index f984c03a0..bcdaf2d05 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- import json import logging - +import os.path import requests from .alerts import Alerter @@ -33,6 +33,7 @@ def __init__(self, *args): self.alias = self.rule.get('opsgenie_alias') self.opsgenie_proxy = self.rule.get('opsgenie_proxy', None) self.priority = self.rule.get('opsgenie_priority') + self.opsgenie_details = self.rule.get('opsgenie_details', {}) def _parse_responders(self, responders, responder_args, matches, default_responders): if responder_args: @@ -97,6 +98,10 @@ def alert(self, matches): if self.alias is not None: post['alias'] = self.alias.format(**matches[0]) + details = self.get_details(matches) + if details: + post['details'] = details + logging.debug(json.dumps(post)) headers = { @@ -162,3 +167,19 @@ def get_info(self): if self.teams: ret['teams'] = self.teams return ret + + def get_details(self, matches): + details = {} + + for key, value in self.opsgenie_details.items(): + + if type(value) is dict: + if 'field' in value: + field_value = lookup_es_key(matches[0], value['field']) + if field_value is not None: + details[key] = str(field_value) + + elif type(value) is str: + details[key] = os.path.expandvars(value) + + return details diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 6cdad648e..b940acd47 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -298,6 +298,20 @@ properties: mattermost_msg_pretext: {type: string} mattermost_msg_fields: *mattermostField + ## Opsgenie + opsgenie_details: + type: object + minProperties: 1 + patternProperties: + "^.+$": + oneOf: + - type: string + - type: object + additionalProperties: false + required: [field] + properties: + field: {type: string, minLength: 1} + ### PagerDuty pagerduty_service_key: {type: string} pagerduty_client_name: {type: string} diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 549b60624..e3a897e5f 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -456,6 +456,261 @@ def test_opsgenie_default_alert_routing(): assert alert.get_info()['recipients'] == ['devops@test.com'] +def test_opsgenie_details_with_constant_value(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': 'Bar'} + } + match = { + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': {'field': 'message'}} + } + match = { + 'message': 'Bar', + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_nested_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': {'field': 'nested.field'}} + } + match = { + 'nested': { + 'field': 'Bar' + }, + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_non_string_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': { + 'Age': {'field': 'age'}, + 'Message': {'field': 'message'} + } + } + match = { + 'age': 10, + 'message': { + 'format': 'The cow goes %s!', + 'arg0': 'moo' + } + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': { + 'Age': '10', + 'Message': "{'format': 'The cow goes %s!', 'arg0': 'moo'}" + }, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_missing_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': { + 'Message': {'field': 'message'}, + 'Missing': {'field': 'missing'} + } + } + match = { + 'message': 'Testing', + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Message': 'Testing'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_environment_variable_replacement(environ): + environ.update({ + 'TEST_VAR': 'Bar' + }) + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': '$TEST_VAR'} + } + match = { + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + def test_jira(): description_txt = "Description stuff goes here like a runbook link." rule = { From 511538bba862dac66ae390d08903bcc5ff68fa00 Mon Sep 17 00:00:00 2001 From: Paul Gardiner Date: Mon, 18 Nov 2019 11:59:15 -0500 Subject: [PATCH 246/264] updated rules_mapping to include spike_aggregation --- elastalert/loaders.py | 1 + 1 file changed, 1 insertion(+) diff --git a/elastalert/loaders.py b/elastalert/loaders.py index c602d0dc5..771194768 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -50,6 +50,7 @@ class RulesLoader(object): 'cardinality': ruletypes.CardinalityRule, 'metric_aggregation': ruletypes.MetricAggregationRule, 'percentage_match': ruletypes.PercentageMatchRule, + 'spike_aggregation': ruletypes.SpikeMetricAggregationRule, } # Used to map names of alerts to their classes From 1e7082ba4550391c2481fc10e5ffc2de8c8e76f3 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 5 Dec 2019 12:39:16 -0800 Subject: [PATCH 247/264] Allow run_every to be unique per rule --- elastalert/elastalert.py | 9 ++++----- tests/base_test.py | 29 ++++++++++++++++++++--------- tests/conftest.py | 1 + 3 files changed, 25 insertions(+), 14 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 30ef4a2a7..24b10ced9 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -27,10 +27,10 @@ from elasticsearch.exceptions import TransportError from . import kibana -from .kibana_discover import generate_kibana_discover_url from .alerts import DebugAlerter from .config import load_conf from .enhancements import DropMatchException +from .kibana_discover import generate_kibana_discover_url from .ruletypes import FlatlineRule from .util import add_raw_postfix from .util import cronite_datetime_to_timestamp @@ -1022,8 +1022,7 @@ def init_rule(self, new_rule, new=True): 'processed_hits', 'starttime', 'minimum_starttime', - 'has_run_once', - 'run_every'] + 'has_run_once'] for prop in copy_properties: if prop not in rule: continue @@ -1467,8 +1466,8 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): # Compute top count keys if rule.get('top_count_keys'): for match in matches: - if 'query_key' in rule and rule['query_key'] in match: - qk = match[rule['query_key']] + if 'query_key' in rule: + qk = lookup_es_key(match, rule['query_key']) else: qk = None diff --git a/tests/base_test.py b/tests/base_test.py index 0e57a2ff9..92dc35f7e 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -75,7 +75,7 @@ def test_init_rule(ea): ea.rules[0]['starttime'] = '2014-01-02T00:11:22' ea.rules[0]['processed_hits'] = ['abcdefg'] new_rule = ea.init_rule(new_rule, False) - for prop in ['starttime', 'agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime']: + for prop in ['starttime', 'agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime', 'run_every']: assert new_rule[prop] == ea.rules[0][prop] # Properties are fresh @@ -84,6 +84,11 @@ def test_init_rule(ea): assert 'starttime' not in new_rule assert new_rule['processed_hits'] == {} + # Assert run_every is unique + new_rule['run_every'] = datetime.timedelta(seconds=17) + new_rule = ea.init_rule(new_rule, True) + assert new_rule['run_every'] == datetime.timedelta(seconds=17) + def test_query(ea): ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} @@ -989,8 +994,11 @@ def test_kibana_dashboard(ea): def test_rule_changes(ea): ea.rule_hashes = {'rules/rule1.yaml': 'ABC', 'rules/rule2.yaml': 'DEF'} - ea.rules = [ea.init_rule(rule, True) for rule in [{'rule_file': 'rules/rule1.yaml', 'name': 'rule1', 'filter': []}, - {'rule_file': 'rules/rule2.yaml', 'name': 'rule2', 'filter': []}]] + run_every = datetime.timedelta(seconds=1) + ea.rules = [ea.init_rule(rule, True) for rule in [{'rule_file': 'rules/rule1.yaml', 'name': 'rule1', 'filter': [], + 'run_every': run_every}, + {'rule_file': 'rules/rule2.yaml', 'name': 'rule2', 'filter': [], + 'run_every': run_every}]] ea.rules[1]['processed_hits'] = ['save me'] new_hashes = {'rules/rule1.yaml': 'ABC', 'rules/rule3.yaml': 'XXX', @@ -998,8 +1006,8 @@ def test_rule_changes(ea): with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: - mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml'}, - {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml'}] + mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml', 'run_every': run_every}, + {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml', 'run_every': run_every}] mock_hashes.return_value = new_hashes ea.load_rule_changes() @@ -1021,7 +1029,7 @@ def test_rule_changes(ea): with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: with mock.patch.object(ea, 'send_notification_email') as mock_send: mock_load.return_value = {'filter': [], 'name': 'rule3', 'new': 'stuff', - 'rule_file': 'rules/rule4.yaml'} + 'rule_file': 'rules/rule4.yaml', 'run_every': run_every} mock_hashes.return_value = new_hashes ea.load_rule_changes() mock_send.assert_called_once_with(exception=mock.ANY, rule_file='rules/rule4.yaml') @@ -1033,7 +1041,8 @@ def test_rule_changes(ea): new_hashes.update({'rules/rule4.yaml': 'asdf'}) with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: - mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, 'rule_file': 'rules/rule4.yaml'} + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, + 'rule_file': 'rules/rule4.yaml', 'run_every': run_every} mock_hashes.return_value = new_hashes ea.load_rule_changes() assert len(ea.rules) == 3 @@ -1044,7 +1053,8 @@ def test_rule_changes(ea): new_hashes['rules/rule4.yaml'] = 'qwerty' with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: - mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml', + 'run_every': run_every} mock_hashes.return_value = new_hashes ea.load_rule_changes() assert len(ea.rules) == 4 @@ -1053,7 +1063,8 @@ def test_rule_changes(ea): new_hashes.pop('rules/rule4.yaml') with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: - mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml', + 'run_every': run_every} mock_hashes.return_value = new_hashes ea.load_rule_changes() ea.scheduler.remove_job.assert_called_with(job_id='rule4') diff --git a/tests/conftest.py b/tests/conftest.py index 2b547ba41..6844296ee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -202,6 +202,7 @@ def ea_sixsix(): 'index': 'idx', 'filter': [], 'include': ['@timestamp'], + 'run_every': datetime.timedelta(seconds=1), 'aggregation': datetime.timedelta(0), 'realert': datetime.timedelta(0), 'processed_hits': {}, From bdbe144f8d95c30365002711bbedfb4e8f7da82b Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 5 Dec 2019 14:43:21 -0800 Subject: [PATCH 248/264] Pin elasticsearch to 7.0.0 in setup.py --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index ac6506ae2..2bade3e1e 100644 --- a/setup.py +++ b/setup.py @@ -33,7 +33,7 @@ 'boto3>=1.4.4', 'configparser>=3.5.0', 'croniter>=0.3.16', - 'elasticsearch>=7.0.0', + 'elasticsearch==7.0.0', 'envparse>=0.2.0', 'exotel>=0.1.3', 'jira>=2.0.0', From ec5d03b95708ea0aa3d29c065f9794fdd95a82a1 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 2 Jan 2020 11:22:36 -0800 Subject: [PATCH 249/264] Pin covereage version --- requirements-dev.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-dev.txt b/requirements-dev.txt index 1cb67cb8e..558761d9e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,5 +1,5 @@ -r requirements.txt -coverage +coverage==4.5.4 flake8 pre-commit pylint<1.4 From 5411c8c064c62e7f7102b51b5e559179469e3114 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 14 Apr 2020 14:12:19 -0700 Subject: [PATCH 250/264] Removed thehive alerter --- docs/source/elastalert.rst | 1 - docs/source/ruletypes.rst | 47 +--------------------- elastalert/alerts.py | 81 -------------------------------------- elastalert/loaders.py | 1 - requirements.txt | 1 - setup.py | 3 +- 6 files changed, 2 insertions(+), 132 deletions(-) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index 970cde7aa..c29f80f23 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -42,7 +42,6 @@ Currently, we have support built in for these alert types: - GoogleChat - Debug - Stomp -- theHive Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 8d9644060..790c85919 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -549,7 +549,7 @@ kibana_discover_version ``kibana_discover_version``: Specifies the version of the Kibana Discover application. -The currently supported versions of Kibana Discover are: +The currently supported versions of Kibana Discover are: - `5.6` - `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` @@ -2186,51 +2186,6 @@ Required: ``linenotify_access_token``: The access token that you got from https://notify-bot.line.me/my/ -theHive -~~~~~~~ - -theHive alert type will send JSON request to theHive (Security Incident Response Platform) with TheHive4py API. Sent request will be stored like Hive Alert with description and observables. - -Required: - -``hive_connection``: The connection details as key:values. Required keys are ``hive_host``, ``hive_port`` and ``hive_apikey``. - -``hive_alert_config``: Configuration options for the alert. - -Optional: - -``hive_proxies``: Proxy configuration. - -``hive_observable_data_mapping``: If needed, matched data fields can be mapped to TheHive observable types using python string formatting. - -Example usage:: - - alert: hivealerter - - hive_connection: - hive_host: http://localhost - hive_port: - hive_apikey: - hive_proxies: - http: '' - https: '' - - hive_alert_config: - title: 'Title' ## This will default to {rule[index]_rule[name]} if not provided - type: 'external' - source: 'elastalert' - description: '{match[field1]} {rule[name]} Sample description' - severity: 2 - tags: ['tag1', 'tag2 {rule[name]}'] - tlp: 3 - status: 'New' - follow: True - - hive_observable_data_mapping: - - domain: "{match[field1]}_{rule[name]}" - - domain: "{match[field]}" - - ip: "{match[ip_field]}" - Zabbix ~~~~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index f5ca22070..f8ea9af34 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -4,7 +4,6 @@ import json import logging import os -import re import subprocess import sys import time @@ -29,10 +28,6 @@ from requests.exceptions import RequestException from staticconf.loader import yaml_loader from texttable import Texttable -from thehive4py.api import TheHiveApi -from thehive4py.models import Alert -from thehive4py.models import AlertArtifact -from thehive4py.models import CustomFieldHelper from twilio.base.exceptions import TwilioRestException from twilio.rest import Client as TwilioClient @@ -2109,79 +2104,3 @@ def alert(self, matches): def get_info(self): return {"type": "linenotify", "linenotify_access_token": self.linenotify_access_token} - - -class HiveAlerter(Alerter): - """ - Use matched data to create alerts containing observables in an instance of TheHive - """ - - required_options = set(['hive_connection', 'hive_alert_config']) - - def alert(self, matches): - - connection_details = self.rule['hive_connection'] - - api = TheHiveApi( - connection_details.get('hive_host'), - connection_details.get('hive_apikey', ''), - proxies=connection_details.get('hive_proxies', {'http': '', 'https': ''}), - cert=connection_details.get('hive_verify', False)) - - for match in matches: - context = {'rule': self.rule, 'match': match} - - artifacts = [] - for mapping in self.rule.get('hive_observable_data_mapping', []): - for observable_type, match_data_key in mapping.items(): - try: - match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) - rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) - data_keys = match_data_keys + rule_data_keys - context_keys = list(context['match'].keys()) + list(context['rule'].keys()) - if all([True if k in context_keys else False for k in data_keys]): - artifacts.append(AlertArtifact(dataType=observable_type, data=match_data_key.format(**context))) - except KeyError: - raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) - - alert_config = { - 'artifacts': artifacts, - 'sourceRef': str(uuid.uuid4())[0:6], - 'title': '{rule[index]}_{rule[name]}'.format(**context) - } - alert_config.update(self.rule.get('hive_alert_config', {})) - - for alert_config_field, alert_config_value in alert_config.items(): - if alert_config_field == 'customFields': - custom_fields = CustomFieldHelper() - for cf_key, cf_value in alert_config_value.items(): - try: - func = getattr(custom_fields, 'add_{}'.format(cf_value['type'])) - except AttributeError: - raise Exception('unsupported custom field type {}'.format(cf_value['type'])) - value = cf_value['value'].format(**context) - func(cf_key, value) - alert_config[alert_config_field] = custom_fields.build() - elif isinstance(alert_config_value, str): - alert_config[alert_config_field] = alert_config_value.format(**context) - elif isinstance(alert_config_value, (list, tuple)): - formatted_list = [] - for element in alert_config_value: - try: - formatted_list.append(element.format(**context)) - except (AttributeError, KeyError, IndexError): - formatted_list.append(element) - alert_config[alert_config_field] = formatted_list - - alert = Alert(**alert_config) - response = api.create_alert(alert) - - if response.status_code != 201: - raise Exception('alert not successfully created in TheHive\n{}'.format(response.text)) - - def get_info(self): - - return { - 'type': 'hivealerter', - 'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '') - } diff --git a/elastalert/loaders.py b/elastalert/loaders.py index 771194768..f15e5f2a2 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -77,7 +77,6 @@ class RulesLoader(object): 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, 'post': alerts.HTTPPostAlerter, - 'hivealerter': alerts.HiveAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list diff --git a/requirements.txt b/requirements.txt index ce392cb18..c66ca8d79 100644 --- a/requirements.txt +++ b/requirements.txt @@ -20,5 +20,4 @@ PyYAML>=5.1 requests>=2.0.0 stomp.py>=4.1.17 texttable>=0.8.8 -thehive4py>=1.4.4 twilio==6.0.0 diff --git a/setup.py b/setup.py index 2bade3e1e..74cca0f11 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.2.1', + version='0.2.2', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', @@ -47,7 +47,6 @@ 'stomp.py>=4.1.17', 'texttable>=0.8.8', 'twilio>=6.0.0,<6.1', - 'thehive4py>=1.4.4', 'python-magic>=0.4.15', 'cffi>=1.11.5' ] From b45d767b1a5038e1dc7c569dd77b280d4deaa81b Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Tue, 14 Apr 2020 15:07:48 -0700 Subject: [PATCH 251/264] Updated changelog for 0.2.2 --- changelog.md | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/changelog.md b/changelog.md index fe30b573c..2e054fdcb 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,23 @@ # Change Log +# v0.2.2 + +# Added +- Integration with Kibana Discover app +- Addied ability to specify opsgenie alert details  + +### Fixed +- Fix some encoding issues with command alerter +- Better error messages for missing config file +- Fixed an issue with run_every not applying per-rule +- Fixed an issue with rules not being removed +- Fixed an issue with top count keys and nested query keys +- Various documentation fixes +- Fixed an issue with not being able to use spike aggregation + +### Removed +- Remove The Hive alerter + # v0.2.1 ### Fixed From 6a4ae2dfe964b28c134ec6ea5d851a6e0b2fa474 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 15 Apr 2020 14:17:19 -0700 Subject: [PATCH 252/264] Re-add TheHive alerter without any libraries --- docs/source/elastalert.rst | 1 + docs/source/ruletypes.rst | 45 ++++++++++++++++++++++++ elastalert/alerts.py | 70 ++++++++++++++++++++++++++++++++++++++ elastalert/loaders.py | 1 + 4 files changed, 117 insertions(+) diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index c29f80f23..b1008c3c4 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -42,6 +42,7 @@ Currently, we have support built in for these alert types: - GoogleChat - Debug - Stomp +- TheHive Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index 790c85919..ea415f0f2 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2186,6 +2186,51 @@ Required: ``linenotify_access_token``: The access token that you got from https://notify-bot.line.me/my/ +theHive +~~~~~~ + +theHive alert type will send JSON request to theHive (Security Incident Response Platform) with TheHive4py API. Sent request will be stored like Hive Alert with description and observables. + +Required: + +``hive_connection``: The connection details as key:values. Required keys are ``hive_host``, ``hive_port`` and ``hive_apikey``. + +``hive_alert_config``: Configuration options for the alert. + +Optional: + +``hive_proxies``: Proxy configuration. + +``hive_observable_data_mapping``: If needed, matched data fields can be mapped to TheHive observable types using python string formatting. + +Example usage:: + + alert: hivealerter + + hive_connection: + hive_host: http://localhost + hive_port: + hive_apikey: + hive_proxies: + http: '' + https: '' + + hive_alert_config: + title: 'Title' ## This will default to {rule[index]_rule[name]} if not provided + type: 'external' + source: 'elastalert' + description: '{match[field1]} {rule[name]} Sample description' + severity: 2 + tags: ['tag1', 'tag2 {rule[name]}'] + tlp: 3 + status: 'New' + follow: True + + hive_observable_data_mapping: + - domain: "{match[field1]}_{rule[name]}" + - domain: "{match[field]}" + - ip: "{match[ip_field]}" + Zabbix ~~~~~~~~~~~ diff --git a/elastalert/alerts.py b/elastalert/alerts.py index f8ea9af34..8fa59542e 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -4,6 +4,7 @@ import json import logging import os +import re import subprocess import sys import time @@ -2104,3 +2105,72 @@ def alert(self, matches): def get_info(self): return {"type": "linenotify", "linenotify_access_token": self.linenotify_access_token} + + +class HiveAlerter(Alerter): + """ + Use matched data to create alerts containing observables in an instance of TheHive + """ + + required_options = set(['hive_connection', 'hive_alert_config']) + + def alert(self, matches): + + connection_details = self.rule['hive_connection'] + + for match in matches: + context = {'rule': self.rule, 'match': match} + + artifacts = [] + for mapping in self.rule.get('hive_observable_data_mapping', []): + for observable_type, match_data_key in mapping.items(): + try: + match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) + rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) + data_keys = match_data_keys + rule_data_keys + context_keys = list(context['match'].keys()) + list(context['rule'].keys()) + if all([True if k in context_keys else False for k in data_keys]): + artifact = {'tlp': 2, 'tags': [], 'message': None, 'dataType': observable_type, + 'data': match_data_key.format(**context)} + artifacts.append(artifact) + except KeyError: + raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) + + alert_config = { + 'artifacts': artifacts, + 'sourceRef': str(uuid.uuid4())[0:6], + 'customFields': {}, + 'caseTemplate': None, + 'title': '{rule[index]}_{rule[name]}'.format(**context), + 'date': int(time.time()) * 1000 + } + alert_config.update(self.rule.get('hive_alert_config', {})) + + for alert_config_field, alert_config_value in alert_config.items(): + if isinstance(alert_config_value, str): + alert_config[alert_config_field] = alert_config_value.format(**context) + elif isinstance(alert_config_value, (list, tuple)): + formatted_list = [] + for element in alert_config_value: + try: + formatted_list.append(element.format(**context)) + except (AttributeError, KeyError, IndexError): + formatted_list.append(element) + alert_config[alert_config_field] = formatted_list + + alert_body = json.dumps(alert_config, indent=4, sort_keys=True) + req = '{}:{}/api/alert'.format(connection_details['hive_host'], connection_details['hive_port']) + headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer {}'.format(connection_details.get('hive_apikey', ''))} + proxies = connection_details.get('hive_proxies', {'http': '', 'https': ''}) + verify = connection_details.get('hive_verify', False) + response = requests.post(req, headers=headers, data=alert_body, proxies=proxies, verify=verify) + + if response.status_code != 201: + raise Exception('alert not successfully created in TheHive\n{}'.format(response.text)) + + def get_info(self): + + return { + 'type': 'hivealerter', + 'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '') + } diff --git a/elastalert/loaders.py b/elastalert/loaders.py index f15e5f2a2..771194768 100644 --- a/elastalert/loaders.py +++ b/elastalert/loaders.py @@ -77,6 +77,7 @@ class RulesLoader(object): 'servicenow': alerts.ServiceNowAlerter, 'alerta': alerts.AlertaAlerter, 'post': alerts.HTTPPostAlerter, + 'hivealerter': alerts.HiveAlerter } # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list From 4ebdabfc4d7705520fb823607a481237f0663ff1 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 15 Apr 2020 14:45:12 -0700 Subject: [PATCH 253/264] version 0.2.3 --- changelog.md | 7 ++++++- setup.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/changelog.md b/changelog.md index 2e054fdcb..6e3b56f6b 100644 --- a/changelog.md +++ b/changelog.md @@ -1,8 +1,13 @@ # Change Log +# v0.2.3 + +### Added +- Added back TheHive alerter without TheHive4py library + # v0.2.2 -# Added +### Added - Integration with Kibana Discover app - Addied ability to specify opsgenie alert details  diff --git a/setup.py b/setup.py index 74cca0f11..9aed0ac85 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.2.2', + version='0.2.3', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From 6d5edf077bc80ccdae17593166633ce7a674af32 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Wed, 15 Apr 2020 15:57:42 -0700 Subject: [PATCH 254/264] Fix documentation --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index ea415f0f2..a947a77b7 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -2187,7 +2187,7 @@ Required: ``linenotify_access_token``: The access token that you got from https://notify-bot.line.me/my/ theHive -~~~~~~ +~~~~~~~ theHive alert type will send JSON request to theHive (Security Incident Response Platform) with TheHive4py API. Sent request will be stored like Hive Alert with description and observables. From cd6d5555774f93a5becd272ff37765efdcbd1ce9 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 16 Apr 2020 15:10:08 -0700 Subject: [PATCH 255/264] Added back custom fields to the hive alerter --- elastalert/alerts.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 8fa59542e..a453081fb 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -2145,9 +2145,15 @@ def alert(self, matches): 'date': int(time.time()) * 1000 } alert_config.update(self.rule.get('hive_alert_config', {})) - + custom_fields = {} for alert_config_field, alert_config_value in alert_config.items(): - if isinstance(alert_config_value, str): + if alert_config_field == 'customFields': + n = 0 + for cf_key, cf_value in alert_config_value.items(): + cf = {'order': n, cf_value['type']: cf_value['value'].format(**context)} + n += 1 + custom_fields[cf_key] = cf + elif isinstance(alert_config_value, str): alert_config[alert_config_field] = alert_config_value.format(**context) elif isinstance(alert_config_value, (list, tuple)): formatted_list = [] @@ -2157,6 +2163,8 @@ def alert(self, matches): except (AttributeError, KeyError, IndexError): formatted_list.append(element) alert_config[alert_config_field] = formatted_list + if custom_fields: + alert_config['customFields'] = custom_fields alert_body = json.dumps(alert_config, indent=4, sort_keys=True) req = '{}:{}/api/alert'.format(connection_details['hive_host'], connection_details['hive_port']) From 7d369e41fb00127ac278a911dd8025906be46738 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 16 Apr 2020 15:40:53 -0700 Subject: [PATCH 256/264] Version 0.2.4 --- changelog.md | 5 +++++ setup.py | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/changelog.md b/changelog.md index 6e3b56f6b..975d6855f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,10 @@ # Change Log +# v0.2.4 + +### Added +- Added back customFields support for The Hive + # v0.2.3 ### Added diff --git a/setup.py b/setup.py index 9aed0ac85..30ef9495f 100644 --- a/setup.py +++ b/setup.py @@ -8,7 +8,7 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.2.3', + version='0.2.4', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', From 8205d5393c8214e4a4100d45ba52a04ff01251ff Mon Sep 17 00:00:00 2001 From: Feroz Date: Sun, 3 May 2020 17:03:59 +0100 Subject: [PATCH 257/264] Update elastalert-test-rule to use ascii_letters --- elastalert/test_rule.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index ee467031b..06100aa0f 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -252,7 +252,7 @@ def run_elastalert(self, rule, conf, args): used_ids = [] def get_id(): - _id = ''.join([random.choice(string.letters) for i in range(16)]) + _id = ''.join([random.choice(string.ascii_letters) for i in range(16)]) if _id in used_ids: return get_id() used_ids.append(_id) From d715163c2f81f41127f6879bd00d74e003b80ae9 Mon Sep 17 00:00:00 2001 From: Abhishek Jaisingh Date: Tue, 26 May 2020 13:06:11 +0530 Subject: [PATCH 258/264] Fix Flake Lint Ambiguous Variable Error --- elastalert/alerts.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index a453081fb..d3fa7518f 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -794,9 +794,9 @@ def alert(self, matches): except JIRAError as e: logging.exception("Error while commenting on ticket %s: %s" % (ticket, e)) if self.labels: - for l in self.labels: + for label in self.labels: try: - ticket.fields.labels.append(l) + ticket.fields.labels.append(label) except JIRAError as e: logging.exception("Error while appending labels to ticket %s: %s" % (ticket, e)) if self.transition: From 11cd9bd3c437f88b56bc23b3d63455df99ee2fbf Mon Sep 17 00:00:00 2001 From: Varun Vora Date: Wed, 29 Jul 2020 23:33:56 +0530 Subject: [PATCH 259/264] Fix slack kibana properties Made the schema valid according to draft-07 --- elastalert/schema.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index cc5d52395..1241315dc 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -286,9 +286,9 @@ properties: slack_text_string: {type: string} slack_ignore_ssl_errors: {type: boolean} slack_ca_certs: {type: string} - slack_attach_kibana_discover_url {type: boolean} - slack_kibana_discover_color {type: string} - slack_kibana_discover_title {type: string} + slack_attach_kibana_discover_url: {type: boolean} + slack_kibana_discover_color: {type: string} + slack_kibana_discover_title: {type: string} ### Mattermost mattermost_webhook_url: *arrayOfString From 39002929a8229d0a76702555a232f05b8e598da4 Mon Sep 17 00:00:00 2001 From: nsano-rururu Date: Sun, 30 Aug 2020 03:25:23 +0900 Subject: [PATCH 260/264] Fix exotelto docs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit exotel_accout_sid → exotel_account_sid --- docs/source/ruletypes.rst | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index a947a77b7..ff3763712 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -1931,7 +1931,7 @@ Developers in India can use Exotel alerter, it will trigger an incident to a mob The alerter requires the following option: -``exotel_accout_sid``: This is sid of your Exotel account. +``exotel_account_sid``: This is sid of your Exotel account. ``exotel_auth_token``: Auth token assosiated with your Exotel account. From cfad1cc34d11ecdaca008060e612f799e526c6ef Mon Sep 17 00:00:00 2001 From: singyiu Date: Mon, 31 Aug 2020 16:50:45 -0700 Subject: [PATCH 261/264] fix issue 2518, where false positive may be triggered sometimes --- elastalert/elastalert.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 24b10ced9..b078c86db 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -652,7 +652,8 @@ def run_query(self, rule, start=None, end=None, scroll=False): try: if rule.get('scroll_id') and self.thread_data.num_hits < self.thread_data.total_hits and should_scrolling_continue(rule): - self.run_query(rule, start, end, scroll=True) + if not self.run_query(rule, start, end, scroll=True): + return False except RuntimeError: # It's possible to scroll far enough to hit max recursive depth pass @@ -894,7 +895,8 @@ def run_rule(self, rule, endtime, starttime=None): if rule.get('aggregation_query_element'): if endtime - tmp_endtime == segment_size: - self.run_query(rule, tmp_endtime, endtime) + if not self.run_query(rule, tmp_endtime, endtime): + return 0 self.thread_data.cumulative_hits += self.thread_data.num_hits elif total_seconds(rule['original_starttime'] - tmp_endtime) == 0: rule['starttime'] = rule['original_starttime'] From ea62cf4bd1a62be3616f40992d088e58e2727f40 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Thu, 10 Sep 2020 18:05:31 -0700 Subject: [PATCH 262/264] remove python-magic requirement --- requirements.txt | 1 - setup.py | 1 - 2 files changed, 2 deletions(-) diff --git a/requirements.txt b/requirements.txt index c66ca8d79..9c32052d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -15,7 +15,6 @@ prison>=0.1.2 py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 -python-magic>=0.4.15 PyYAML>=5.1 requests>=2.0.0 stomp.py>=4.1.17 diff --git a/setup.py b/setup.py index 30ef9495f..2845836a7 100644 --- a/setup.py +++ b/setup.py @@ -47,7 +47,6 @@ 'stomp.py>=4.1.17', 'texttable>=0.8.8', 'twilio>=6.0.0,<6.1', - 'python-magic>=0.4.15', 'cffi>=1.11.5' ] ) From 734caf2bb55ea9e4c086d6e1bcacf5dbe8710455 Mon Sep 17 00:00:00 2001 From: Feroz Date: Mon, 5 Oct 2020 11:29:50 +0100 Subject: [PATCH 263/264] Fix the use of timestamps in the Jira ticket titles This has come about because we currently use records which have timestamps like: ``` "@timestamp": { "min": "2020-10-03T07:05:01.987Z", "max": "2020-10-05T07:05:09.572Z" }, ``` These work fine with the `timestamp_field` set to `@timestamp.min`. However, when we try to create a Jira alert using ElastAlert from these, we run into a Python error: ``` File "/usr/local/lib/python3.6/site-packages/elastalert/alerts.py", line 875, in create_default_title, title += ' - %s' % (pretty_ts(matches[0][self.rule['timestamp_field']], self.rule.get('use_local_time'))), KeyError: '@timestamp.min' ``` This is because matches[0][self.rule['timestamp_field']] attempts to access the `timestamp_field` directly rather than using a `get()`. The proposed fix will not change any existing behaviour, but will skip the addition of a timestamp to the ticket title if the required field doesn't exist, rather than throwing an error and disabling the rule. --- elastalert/alerts.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/elastalert/alerts.py b/elastalert/alerts.py index d3fa7518f..f2f31853f 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -868,7 +868,9 @@ def create_default_title(self, matches, for_search=False): if for_search: return title - title += ' - %s' % (pretty_ts(matches[0][self.rule['timestamp_field']], self.rule.get('use_local_time'))) + timestamp = matches[0].get(self.rule['timestamp_field']) + if timestamp: + title += ' - %s' % (pretty_ts(timestamp, self.rule.get('use_local_time'))) # Add count for spikes count = matches[0].get('spike_count') From e0bbcb5b71e9fdb4a750c3871d365a882ff17b16 Mon Sep 17 00:00:00 2001 From: Quentin Long Date: Fri, 11 Nov 2022 16:58:08 -0800 Subject: [PATCH 264/264] Link to elastalert2 --- README.md | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 99acc02e7..20a3d5d27 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,5 @@ -Recent changes: As of Elastalert 0.2.0, you must use Python 3.6. Python 2 will not longer be supported. +**ElastAlert is no longer maintained. Please use [ElastAlert2](https://github.com/jertel/elastalert2) instead.** + [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)