Merge branch 'master' into patch-3

Author: Qmando

Dockerfile-test

@@ -1,9 +1,9 @@
 FROM ubuntu:latest
 
 RUN apt-get update && apt-get upgrade -y
-RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox python-pip
+RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git
 
 WORKDIR /home/elastalert
 
 ADD requirements*.txt ./
-RUN pip install -r requirements-dev.txt
+RUN pip3 install -r requirements-dev.txt

docs/source/elastalert.rst

@@ -203,7 +203,6 @@ The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticse
 
 ``skip_invalid``: If ``True``, skip invalid files instead of exiting.
 
-=======
 Logging
 -------
 

docs/source/recipes/writing_filters.rst

@@ -57,7 +57,7 @@ a field that appears to have the value "foo bar", unless it is not analyzed. Con
 matching on analyzed fields, use query_string. See https://www.elastic.co/guide/en/elasticsearch/guide/current/term-vs-full-text.html
 
 `terms <https://www.elastic.co/guide/en/elasticsearch/reference/current/query-dsl-terms-query.html>`_
-*****
+*****************************************************************************************************
 
 
 

docs/source/ruletypes.rst

@@ -58,6 +58,20 @@ Rule Configuration Cheat Sheet
 +--------------------------------------------------------------+           |
 | ``kibana4_end_timedelta`` (time, default: 10 min)            |           |
 +--------------------------------------------------------------+           |
+| ``generate_kibana_discover_url`` (boolean, default False)    |           |
++--------------------------------------------------------------+           |
+| ``kibana_discover_app_url`` (string, no default)             |           |
++--------------------------------------------------------------+           |
+| ``kibana_discover_version`` (string, no default)             |           |
++--------------------------------------------------------------+           |
+| ``kibana_discover_index_pattern_id`` (string, no default)    |           |
++--------------------------------------------------------------+           |
+| ``kibana_discover_columns`` (list of strs, default _source)  |           |
++--------------------------------------------------------------+           |
+| ``kibana_discover_from_timedelta`` (time, default: 10 min)   |           |
++--------------------------------------------------------------+           |
+| ``kibana_discover_to_timedelta`` (time, default: 10 min)     |           |
++--------------------------------------------------------------+           |
 | ``use_local_time`` (boolean, default True)                   |           |
 +--------------------------------------------------------------+           |
 | ``realert`` (time, default: 1 min)                           |           |
@@ -510,6 +524,85 @@ This value is added in back of the event. For example,
 
 ``kibana4_end_timedelta: minutes: 2``
 
+generate_kibana_discover_url
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``generate_kibana_discover_url``: Enables the generation of the ``kibana_discover_url`` variable for the Kibana Discover application.
+This setting requires the following settings are also configured:
+
+- ``kibana_discover_app_url``
+- ``kibana_discover_version``
+- ``kibana_discover_index_pattern_id``
+
+``generate_kibana_discover_url: true``
+
+kibana_discover_app_url
+^^^^^^^^^^^^^^^^^^^^^^^
+
+``kibana_discover_app_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_url`` variable.
+This value can use `$VAR` and `${VAR}` references to expand environment variables.
+
+``kibana_discover_app_url: http://kibana:5601/#/discover``
+
+kibana_discover_version
+^^^^^^^^^^^^^^^^^^^^^^^
+
+``kibana_discover_version``: Specifies the version of the Kibana Discover application.
+
+The currently supported versions of Kibana Discover are: 
+
+- `5.6`
+- `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8`
+- `7.0`, `7.1`, `7.2`, `7.3`
+
+``kibana_discover_version: '7.3'``
+
+kibana_discover_index_pattern_id
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``kibana_discover_index_pattern_id``: The id of the index pattern to link to in the Kibana Discover application.
+These ids are usually generated and can be found in url of the index pattern management page, or by exporting its saved object.
+
+Example export of an index pattern's saved object:
+
+.. code-block:: text
+
+    [
+        {
+            "_id": "4e97d188-8a45-4418-8a37-07ed69b4d34c",
+            "_type": "index-pattern",
+            "_source": { ... }
+        }
+    ]
+
+You can modify an index pattern's id by exporting the saved object, modifying the ``_id`` field, and re-importing.
+
+``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c``
+
+kibana_discover_columns
+^^^^^^^^^^^^^^^^^^^^^^^
+
+``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link.
+Defaults to the ``_source`` column.
+
+``kibana_discover_columns: [ timestamp, message ]``
+
+kibana_discover_from_timedelta
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``kibana_discover_from_timedelta``:  The offset to the `from` time of the Kibana Discover link's time range.
+The `from` time is calculated by subtracting this timedelta from the event time.  Defaults to 10 minutes.
+
+``kibana_discover_from_timedelta: minutes: 2``
+
+kibana_discover_to_timedelta
+^^^^^^^^^^^^^^^^^^^^^^^^^^^^
+
+``kibana_discover_to_timedelta``:  The offset to the `to` time of the Kibana Discover link's time range.
+The `to` time is calculated by adding this timedelta to the event time.  Defaults to 10 minutes.
+
+``kibana_discover_to_timedelta: minutes: 2``
+
 use_local_time
 ^^^^^^^^^^^^^^
 
@@ -1305,7 +1398,7 @@ With ``alert_text_type: aggregation_summary_only``::
     body                = rule_name
 
                           aggregation_summary
-+
+
 ruletype_text is the string returned by RuleType.get_match_str.
 
 field_values will contain every key value pair included in the results from Elasticsearch. These fields include "@timestamp" (or the value of ``timestamp_field``),
@@ -1689,7 +1782,7 @@ Provide absolute address of the pciture, for example: http://some.address.com/im
 ``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles.
 
 Mattermost
-~~~~~
+~~~~~~~~~~
 
 Mattermost alerter will send a notification to a predefined Mattermost channel. The body of the notification is formatted the same as with other alerters.
 

elastalert/elastalert.py

@@ -23,10 +23,11 @@
 from croniter import croniter
 from elasticsearch.exceptions import ConnectionError
 from elasticsearch.exceptions import ElasticsearchException
-from elasticsearch.exceptions import TransportError
 from elasticsearch.exceptions import NotFoundError
+from elasticsearch.exceptions import TransportError
 
 from . import kibana
+from .kibana_discover import generate_kibana_discover_url
 from .alerts import DebugAlerter
 from .config import load_conf
 from .enhancements import DropMatchException
@@ -1067,7 +1068,13 @@ def load_rule_changes(self):
             if rule_file not in new_rule_hashes:
                 # Rule file was deleted
                 elastalert_logger.info('Rule file %s not found, stopping rule execution' % (rule_file))
-                self.rules = [rule for rule in self.rules if rule['rule_file'] != rule_file]
+                for rule in self.rules:
+                    if rule['rule_file'] == rule_file:
+                        break
+                else:
+                    continue
+                self.scheduler.remove_job(job_id=rule['name'])
+                self.rules.remove(rule)
                 continue
             if hash_value != new_rule_hashes[rule_file]:
                 # Rule file was changed, reload rule
@@ -1497,6 +1504,11 @@ def send_alert(self, matches, rule, alert_time=None, retried=False):
             if kb_link:
                 matches[0]['kibana_link'] = kb_link
 
+        if rule.get('generate_kibana_discover_url'):
+            kb_link = generate_kibana_discover_url(rule, matches[0])
+            if kb_link:
+                matches[0]['kibana_discover_url'] = kb_link
+
         # Enhancements were already run at match time if
         # run_enhancements_first is set or
         # retried==True, which means this is a retry of a failed alert

elastalert/kibana_discover.py

@@ -0,0 +1,192 @@
+# -*- coding: utf-8 -*-
+# flake8: noqa
+import datetime
+import logging
+import json
+import os.path
+import prison
+import urllib.parse
+
+from .util import EAException
+from .util import lookup_es_key
+from .util import ts_add
+
+kibana_default_timedelta = datetime.timedelta(minutes=10)
+
+kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8'])
+kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3'])
+
+def generate_kibana_discover_url(rule, match):
+    ''' Creates a link for a kibana discover app. '''
+
+    discover_app_url = rule.get('kibana_discover_app_url')
+    if not discover_app_url:
+        logging.warning(
+            'Missing kibana_discover_app_url for rule %s' % (
+                rule.get('name', '<MISSING NAME>')
+            )
+        )
+        return None
+
+    kibana_version = rule.get('kibana_discover_version')
+    if not kibana_version:
+        logging.warning(
+            'Missing kibana_discover_version for rule %s' % (
+                rule.get('name', '<MISSING NAME>')
+            )
+        )
+        return None
+
+    index = rule.get('kibana_discover_index_pattern_id')
+    if not index:
+        logging.warning(
+            'Missing kibana_discover_index_pattern_id for rule %s' % (
+                rule.get('name', '<MISSING NAME>')
+            )
+        )
+        return None
+
+    columns = rule.get('kibana_discover_columns', ['_source'])
+    filters = rule.get('filter', [])
+
+    if 'query_key' in rule:
+        query_keys = rule.get('compound_query_key', [rule['query_key']])
+    else:
+        query_keys = []
+
+    timestamp = lookup_es_key(match, rule['timestamp_field'])
+    timeframe = rule.get('timeframe', kibana_default_timedelta)
+    from_timedelta = rule.get('kibana_discover_from_timedelta', timeframe)
+    from_time = ts_add(timestamp, -from_timedelta)
+    to_timedelta = rule.get('kibana_discover_to_timedelta', timeframe)
+    to_time = ts_add(timestamp, to_timedelta)
+
+    if kibana_version in kibana5_kibana6_versions:
+        globalState = kibana6_disover_global_state(from_time, to_time)
+        appState = kibana_discover_app_state(index, columns, filters, query_keys, match)
+
+    elif kibana_version in kibana7_versions:
+        globalState = kibana7_disover_global_state(from_time, to_time)
+        appState = kibana_discover_app_state(index, columns, filters, query_keys, match)
+
+    else:
+        logging.warning(
+            'Unknown kibana discover application version %s for rule %s' % (
+                kibana_version,
+                rule.get('name', '<MISSING NAME>')
+            )
+        )
+        return None
+
+    return "%s?_g=%s&_a=%s" % (
+        os.path.expandvars(discover_app_url),
+        urllib.parse.quote(globalState),
+        urllib.parse.quote(appState)
+    )
+
+
+def kibana6_disover_global_state(from_time, to_time):
+    return prison.dumps( {
+        'refreshInterval': {
+            'pause': True,
+            'value': 0
+        },
+        'time': {
+            'from': from_time,
+            'mode': 'absolute',
+            'to': to_time
+        }
+    } )
+
+
+def kibana7_disover_global_state(from_time, to_time):
+    return prison.dumps( {
+        'filters': [],
+        'refreshInterval': {
+            'pause': True,
+            'value': 0
+        },
+        'time': {
+            'from': from_time,
+            'to': to_time
+        }
+    } )
+
+
+def kibana_discover_app_state(index, columns, filters, query_keys, match):
+    app_filters = []
+
+    if filters:
+        bool_filter = { 'must': filters }
+        app_filters.append( {
+            '$state': {
+                'store': 'appState'
+            },
+            'bool': bool_filter,
+            'meta': {
+                'alias': 'filter',
+                'disabled': False,
+                'index': index,
+                'key': 'bool',
+                'negate': False,
+                'type': 'custom',
+                'value': json.dumps(bool_filter, separators=(',', ':'))
+            },
+        } )
+
+    for query_key in query_keys:
+        query_value = lookup_es_key(match, query_key)
+
+        if query_value is None:
+            app_filters.append( {
+                '$state': {
+                    'store': 'appState'
+                },
+                'exists': {
+                    'field': query_key
+                },
+                'meta': {
+                    'alias': None,
+                    'disabled': False,
+                    'index': index,
+                    'key': query_key,
+                    'negate': True,
+                    'type': 'exists',
+                    'value': 'exists'
+                }
+            } )
+
+        else:
+            app_filters.append( {
+                '$state': {
+                    'store': 'appState'
+                },
+                'meta': {
+                    'alias': None,
+                    'disabled': False,
+                    'index': index,
+                    'key': query_key,
+                    'negate': False,
+                    'params': {
+                        'query': query_value,
+                        'type': 'phrase'
+                    },
+                    'type': 'phrase',
+                    'value': str(query_value)
+                },
+                'query': {
+                    'match': {
+                        query_key: {
+                            'query': query_value,
+                            'type': 'phrase'
+                        }
+                    }
+                }
+            } )
+
+    return prison.dumps( {
+        'columns': columns,
+        'filters': app_filters,
+        'index': index,
+        'interval': 'auto'
+    } )