diff --git a/.editorconfig b/.editorconfig new file mode 100644 index 000000000..f8eafe691 --- /dev/null +++ b/.editorconfig @@ -0,0 +1,17 @@ +root = true + +[*] +end_of_line = lf +insert_final_newline = true +charset = utf-8 + +[*.py] +indent_style = space +indent_size = 4 + +[Makefile] +indent_style = tab + +[{*.json,*.yml,*.yaml}] +indent_style = space +indent_size = 2 diff --git a/.travis.yml b/.travis.yml index 3b189d529..569bf12d6 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,12 +1,35 @@ language: python python: -- '2.7' +- '3.6' env: - TOXENV=docs -- TOXENV=py27 +- TOXENV=py36 install: - pip install tox -script: make test +- > + if [[ -n "${ES_VERSION}" ]] ; then + wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz + mkdir elasticsearch-${ES_VERSION} && tar -xzf elasticsearch-${ES_VERSION}.tar.gz -C elasticsearch-${ES_VERSION} --strip-components=1 + ./elasticsearch-${ES_VERSION}/bin/elasticsearch & + fi +script: +- > + if [[ -n "${ES_VERSION}" ]] ; then + wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200 + make test-elasticsearch + else + make test + fi +jobs: + include: + - stage: 'Elasticsearch test' + env: TOXENV=py36 ES_VERSION=7.0.0-linux-x86_64 + - env: TOXENV=py36 ES_VERSION=6.6.2 + - env: TOXENV=py36 ES_VERSION=6.3.2 + - env: TOXENV=py36 ES_VERSION=6.2.4 + - env: TOXENV=py36 ES_VERSION=6.0.1 + - env: TOXENV=py36 ES_VERSION=5.6.16 + deploy: provider: pypi user: yelplabs diff --git a/Dockerfile-test b/Dockerfile-test index 8005e25f8..3c153e644 100644 --- a/Dockerfile-test +++ b/Dockerfile-test @@ -1,11 +1,9 @@ FROM ubuntu:latest RUN apt-get update && apt-get upgrade -y -RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox - -RUN easy_install pip +RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git WORKDIR /home/elastalert ADD requirements*.txt ./ -RUN pip install -r requirements-dev.txt +RUN pip3 install -r requirements-dev.txt diff --git a/Makefile b/Makefile index 69f590a48..470062ce8 100644 --- a/Makefile +++ b/Makefile @@ -16,6 +16,9 @@ install-hooks: test: tox +test-elasticsearch: + tox -- --runelasticsearch + test-docker: docker-compose --project-name elastalert build tox docker-compose --project-name elastalert run tox diff --git a/README.md b/README.md index f63e25de4..20a3d5d27 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,6 @@ -[![Stories in Ready](https://badge.waffle.io/Yelp/elastalert.png?label=ready&title=Ready)](https://waffle.io/Yelp/elastalert) -[![Stories in In Progress](https://badge.waffle.io/Yelp/elastalert.png?label=in%20progress&title=In%20Progress)](https://waffle.io/Yelp/elastalert) +**ElastAlert is no longer maintained. Please use [ElastAlert2](https://github.com/jertel/elastalert2) instead.** + + [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert) [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge) @@ -47,12 +48,16 @@ Currently, we have built-in support for the following alert types: - MS Teams - Slack - Telegram +- GoogleChat - AWS SNS - VictorOps - PagerDuty +- PagerTree - Exotel - Twilio - Gitter +- Line Notify +- Zabbix Additional rule types and alerts can be easily imported or written. @@ -67,8 +72,23 @@ In addition to this basic usage, there are many other features that make alerts To get started, check out `Running ElastAlert For The First Time` in the [documentation](http://elastalert.readthedocs.org). ## Running ElastAlert +You can either install the latest released version of ElastAlert using pip: + +```pip install elastalert``` + +or you can clone the ElastAlert repository for the most recent changes: + +```git clone https://github.com/Yelp/elastalert.git``` + +Install the module: + +```pip install "setuptools>=11.3"``` + +```python setup.py install``` + +The following invocation can be used to run ElastAlert after installing -``$ python elastalert/elastalert.py [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` +``$ elastalert [--debug] [--verbose] [--start ] [--end ] [--rule ] [--config ]`` ``--debug`` will print additional information to the screen as well as suppresses alerts and instead prints the alert body. Not compatible with `--verbose`. @@ -88,11 +108,11 @@ Eg: ``--rule this_rule.yaml`` ## Third Party Tools And Extras ### Kibana plugin -![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/kibana-elastalert-plugin-showcase.gif) +![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif) Available at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin). ### Docker -A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`. +A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`. ```bash git clone https://github.com/bitsensor/elastalert.git; cd elastalert diff --git a/changelog.md b/changelog.md index 8dde55dfb..975d6855f 100644 --- a/changelog.md +++ b/changelog.md @@ -1,5 +1,120 @@ # Change Log +# v0.2.4 + +### Added +- Added back customFields support for The Hive + +# v0.2.3 + +### Added +- Added back TheHive alerter without TheHive4py library + +# v0.2.2 + +### Added +- Integration with Kibana Discover app +- Addied ability to specify opsgenie alert detailsĀ  + +### Fixed +- Fix some encoding issues with command alerter +- Better error messages for missing config file +- Fixed an issue with run_every not applying per-rule +- Fixed an issue with rules not being removed +- Fixed an issue with top count keys and nested query keys +- Various documentation fixes +- Fixed an issue with not being able to use spike aggregation + +### Removed +- Remove The Hive alerter + +# v0.2.1 + +### Fixed +- Fixed an AttributeError introduced in 0.2.0 + +# v0.2.0 + +- Switched to Python 3 + +### Added +- Add rule loader class for customized rule loading +- Added thread based rules and limit_execution +- Run_every can now be customized per rule + +### Fixed +- Various small fixes + +# v0.1.39 + +### Added +- Added spike alerts for metric aggregations +- Allow SSL connections for Stomp +- Allow limits on alert text length +- Add optional min doc count for terms queries +- Add ability to index into arrays for alert_text_args, etc + +### Fixed +- Fixed bug involving --config flag with create-index +- Fixed some settings not being inherited from the config properly +- Some fixes for Hive alerter +- Close SMTP connections properly +- Fix timestamps in Pagerduty v2 payload +- Fixed an bug causing aggregated alerts to mix up + +# v0.1.38 + +### Added +- Added PagerTree alerter +- Added Line alerter +- Added more customizable logging +- Added new logic in test-rule to detemine the default timeframe + +### Fixed +- Fixed an issue causing buffer_time to sometimes be ignored + +# v0.1.37 + +### Added +- Added more options for Opsgenie alerter +- Added more pagerduty options +- Added ability to add metadata to elastalert logs + +### Fixed +- Fixed some documentation to be more clear +- Stop requiring doc_type for metric aggregations +- No longer puts quotes around regex terms in blacklists or whitelists + +# v0.1.36 + +### Added +- Added a prefix "metric_" to the key used for metric aggregations to avoid possible conflicts +- Added option to skip Alerta certificate validation + +### Fixed +- Fixed a typo in the documentation for spike rule + +# v0.1.35 + +### Fixed +- Fixed an issue preventing new term rule from working with terms query + +# v0.1.34 + +### Added +- Added prefix/suffix support for summary table +- Added support for ignoring SSL validation in Slack +- More visible exceptions during query parse failures + +### Fixed +- Fixed top_count_keys when using compound query_key +- Fixed num_hits sometimes being reported too low +- Fixed an issue with setting ES_USERNAME via env +- Fixed an issue when using test script with custom timestamps +- Fixed a unicode error when using Telegram +- Fixed an issue with jsonschema version conflict +- Fixed an issue with nested timestamps in cardinality type + # v0.1.33 ### Added diff --git a/config.yaml.example b/config.yaml.example index beec38030..9d9176382 100644 --- a/config.yaml.example +++ b/config.yaml.example @@ -57,8 +57,59 @@ es_port: 9200 # This can be a unmapped index, but it is recommended that you run # elastalert-create-index to set a mapping writeback_index: elastalert_status +writeback_alias: elastalert_alerts # If an alert fails for some reason, ElastAlert will retry # sending the alert until this time period has elapsed alert_time_limit: days: 2 + +# Custom logging configuration +# If you want to setup your own logging configuration to log into +# files as well or to Logstash and/or modify log levels, use +# the configuration below and adjust to your needs. +# Note: if you run ElastAlert with --verbose/--debug, the log level of +# the "elastalert" logger is changed to INFO, if not already INFO/DEBUG. +#logging: +# version: 1 +# incremental: false +# disable_existing_loggers: false +# formatters: +# logline: +# format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s' +# +# handlers: +# console: +# class: logging.StreamHandler +# formatter: logline +# level: DEBUG +# stream: ext://sys.stderr +# +# file: +# class : logging.FileHandler +# formatter: logline +# level: DEBUG +# filename: elastalert.log +# +# loggers: +# elastalert: +# level: WARN +# handlers: [] +# propagate: true +# +# elasticsearch: +# level: WARN +# handlers: [] +# propagate: true +# +# elasticsearch.trace: +# level: WARN +# handlers: [] +# propagate: true +# +# '': # root logger +# level: WARN +# handlers: +# - console +# - file +# propagate: false diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst index fd2daec8c..b1008c3c4 100755 --- a/docs/source/elastalert.rst +++ b/docs/source/elastalert.rst @@ -39,8 +39,10 @@ Currently, we have support built in for these alert types: - HipChat - Slack - Telegram +- GoogleChat - Debug - Stomp +- TheHive Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types ` and :ref:`Writing alerts `) @@ -131,9 +133,12 @@ The environment variable ``ES_USE_SSL`` will override this field. ``es_conn_timeout``: Optional; sets timeout for connecting to and reading from ``es_host``; defaults to ``20``. +``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes. +Defaults to ``FileRulesLoader`` if not set. + ``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all files in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload -or remove rules based on their respective config files. +or remove rules based on their respective config files. (only required when using ``FileRulesLoader``). ``scan_subdirectories``: Optional; Sets whether or not ElastAlert should recursively descend the rules directory - ``true`` or ``false``. The default is ``true`` @@ -146,7 +151,11 @@ configuration. ``max_query_size``: The maximum number of documents that will be downloaded from Elasticsearch in a single query. The default is 10,000, and if you expect to get near this number, consider using ``use_count_query`` for the rule. If this -limit is reached, ElastAlert will `scroll `_ through pages the size of ``max_query_size`` until processing all results. +limit is reached, ElastAlert will `scroll `_ +using the size of ``max_query_size`` through the set amount of pages, when ``max_scrolling_count`` is set or until processing all results. + +``max_scrolling_count``: The maximum amount of pages to scroll through. The default is ``0``, which means the scrolling has no limit. +For example if this value is set to ``5`` and the ``max_query_size`` is set to ``10000`` then ``50000`` documents will be downloaded at most. ``scroll_keepalive``: The maximum time (formatted in `Time Units `_) the scrolling context should be kept alive. Avoid using high values as it abuses resources in Elasticsearch, but be mindful to allow sufficient time to finish processing all the results. @@ -161,6 +170,8 @@ from that time, unless it is older than ``old_query_limit``, in which case it wi will upload a traceback message to ``elastalert_metadata`` and if ``notify_email`` is set, send an email notification. The rule will no longer be run until either ElastAlert restarts or the rule file has been modified. This defaults to True. +``show_disabled_rules``: If true, ElastAlert show the disable rules' list when finishes the execution. This defaults to True. + ``notify_email``: An email address, or list of email addresses, to which notification emails will be sent. Currently, only an uncaught exception will send a notification email. The from address, SMTP host, and reply-to header can be set using ``from_addr``, ``smtp_host``, and ``email_reply_to`` options, respectively. By default, no emails will be sent. @@ -188,6 +199,24 @@ The default value is ``False``. Elasticsearch 2.0 - 2.3 does not support dots in ``string_multi_field_name``: If set, the suffix to use for the subfield for string multi-fields in Elasticsearch. The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticsearch 5. +``add_metadata_alert``: If set, alerts will include metadata described in rules (``category``, ``description``, ``owner`` and ``priority``); set to ``True`` or ``False``. The default is ``False``. + +``skip_invalid``: If ``True``, skip invalid files instead of exiting. + +Logging +------- + +By default, ElastAlert uses a simple basic logging configuration to print log messages to standard error. +You can change the log level to ``INFO`` messages by using the ``--verbose`` or ``--debug`` command line options. + +If you need a more sophisticated logging configuration, you can provide a full logging configuration +in the config file. This way you can also configure logging to a file, to Logstash and +adjust the logging format. + +For details, see the end of ``config.yaml.example`` where you can find an example logging +configuration. + + .. _runningelastalert: Running ElastAlert diff --git a/docs/source/index.rst b/docs/source/index.rst index c3f02f535..4219bf13e 100644 --- a/docs/source/index.rst +++ b/docs/source/index.rst @@ -19,6 +19,7 @@ Contents: recipes/adding_alerts recipes/writing_filters recipes/adding_enhancements + recipes/adding_loaders recipes/signing_requests Indices and Tables diff --git a/docs/source/recipes/adding_loaders.rst b/docs/source/recipes/adding_loaders.rst new file mode 100644 index 000000000..672d42390 --- /dev/null +++ b/docs/source/recipes/adding_loaders.rst @@ -0,0 +1,85 @@ +.. _loaders: + +Rules Loaders +======================== + +RulesLoaders are subclasses of ``RulesLoader``, found in ``elastalert/loaders.py``. They are used to +gather rules for a particular source. Your RulesLoader needs to implement three member functions, and +will look something like this: + +.. code-block:: python + + class AwesomeNewRulesLoader(RulesLoader): + def get_names(self, conf, use_rule=None): + ... + def get_hashes(self, conf, use_rule=None): + ... + def get_yaml(self, rule): + ... + +You can import loaders by specifying the type as ``module.file.RulesLoaderName``, where module is the name of a +python module, and file is the name of the python file containing a ``RulesLoader`` subclass named ``RulesLoaderName``. + +Example +------- + +As an example loader, let's retrieve rules from a database rather than from the local file system. First, create a +modules folder for the loader in the ElastAlert directory. + +.. code-block:: console + + $ mkdir elastalert_modules + $ cd elastalert_modules + $ touch __init__.py + +Now, in a file named ``mongo_loader.py``, add + +.. code-block:: python + + from pymongo import MongoClient + from elastalert.loaders import RulesLoader + import yaml + + class MongoRulesLoader(RulesLoader): + def __init__(self, conf): + super(MongoRulesLoader, self).__init__(conf) + self.client = MongoClient(conf['mongo_url']) + self.db = self.client[conf['mongo_db']] + self.cache = {} + + def get_names(self, conf, use_rule=None): + if use_rule: + return [use_rule] + + rules = [] + self.cache = {} + for rule in self.db.rules.find(): + self.cache[rule['name']] = yaml.load(rule['yaml']) + rules.append(rule['name']) + + return rules + + def get_hashes(self, conf, use_rule=None): + if use_rule: + return [use_rule] + + hashes = {} + self.cache = {} + for rule in self.db.rules.find(): + self.cache[rule['name']] = rule['yaml'] + hashes[rule['name']] = rule['hash'] + + return hashes + + def get_yaml(self, rule): + if rule in self.cache: + return self.cache[rule] + + self.cache[rule] = yaml.load(self.db.rules.find_one({'name': rule})['yaml']) + return self.cache[rule] + +Finally, you need to specify in your ElastAlert configuration file that MongoRulesLoader should be used instead of the +default FileRulesLoader, so in your ``elastalert.conf`` file:: + + rules_loader: "elastalert_modules.mongo_loader.MongoRulesLoader" + diff --git a/docs/source/recipes/writing_filters.rst b/docs/source/recipes/writing_filters.rst index 5eff77f84..1d2959262 100644 --- a/docs/source/recipes/writing_filters.rst +++ b/docs/source/recipes/writing_filters.rst @@ -56,22 +56,23 @@ Note that a term query may not behave as expected if a field is analyzed. By def a field that appears to have the value "foo bar", unless it is not analyzed. Conversely, a term query for "foo" will match analyzed strings "foo bar" and "foo baz". For full text matching on analyzed fields, use query_string. See https://www.elastic.co/guide/en/elasticsearch/guide/current/term-vs-full-text.html -terms -***** +`terms `_ +***************************************************************************************************** + + Terms allows for easy combination of multiple term filters:: filter: - terms: - field: ["value1", "value2"] + field: ["value1", "value2"] # value1 OR value2 -Using the minimum_should_match option, you can define a set of term filters of which a certain number must match:: +You can also match on multiple fields:: - terms: fieldX: ["value1", "value2"] fieldY: ["something", "something_else"] fieldZ: ["foo", "bar", "baz"] - minimum_should_match: 2 wildcard ******** @@ -97,7 +98,7 @@ For ranges on fields:: Negation, and, or ***************** -Any of the filters can be embedded in ``not``, ``and``, and ``or``:: +For Elasticsearch 2.X, any of the filters can be embedded in ``not``, ``and``, and ``or``:: filter: - or: @@ -113,6 +114,13 @@ Any of the filters can be embedded in ``not``, ``and``, and ``or``:: term: _type: "something" +For Elasticsearch 5.x, this will not work and to implement boolean logic use query strings:: + + filter: + - query: + query_string: + query: "somefield: somevalue OR foo: bar" + Loading Filters Directly From Kibana 3 -------------------------------------- diff --git a/docs/source/ruletypes.rst b/docs/source/ruletypes.rst index df8934a18..ff3763712 100644 --- a/docs/source/ruletypes.rst +++ b/docs/source/ruletypes.rst @@ -58,6 +58,20 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``kibana4_end_timedelta`` (time, default: 10 min) | | +--------------------------------------------------------------+ | +| ``generate_kibana_discover_url`` (boolean, default False) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_app_url`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_version`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_index_pattern_id`` (string, no default) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_columns`` (list of strs, default _source) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_from_timedelta`` (time, default: 10 min) | | ++--------------------------------------------------------------+ | +| ``kibana_discover_to_timedelta`` (time, default: 10 min) | | ++--------------------------------------------------------------+ | | ``use_local_time`` (boolean, default True) | | +--------------------------------------------------------------+ | | ``realert`` (time, default: 1 min) | | @@ -84,6 +98,8 @@ Rule Configuration Cheat Sheet +--------------------------------------------------------------+ | | ``priority`` (int, default 2) | | +--------------------------------------------------------------+ | +| ``category`` (string, default empty string) | | ++--------------------------------------------------------------+ | | ``scan_entire_timeframe`` (bool, default False) | | +--------------------------------------------------------------+ | | ``import`` (string) | | @@ -406,6 +422,11 @@ priority ``priority``: This value will be used to identify the relative priority of the alert. Optionally, this field can be included in any alert type (e.g. for use in email subject/body text). (Optional, int, default 2) +category +^^^^^^^^ + +``category``: This value will be used to identify the category of the alert. Optionally, this field can be included in any alert type (e.g. for use in email subject/body text). (Optional, string, default empty string) + max_query_size ^^^^^^^^^^^^^^ @@ -503,6 +524,85 @@ This value is added in back of the event. For example, ``kibana4_end_timedelta: minutes: 2`` +generate_kibana_discover_url +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``generate_kibana_discover_url``: Enables the generation of the ``kibana_discover_url`` variable for the Kibana Discover application. +This setting requires the following settings are also configured: + +- ``kibana_discover_app_url`` +- ``kibana_discover_version`` +- ``kibana_discover_index_pattern_id`` + +``generate_kibana_discover_url: true`` + +kibana_discover_app_url +^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_app_url``: The url of the Kibana Discover application used to generate the ``kibana_discover_url`` variable. +This value can use `$VAR` and `${VAR}` references to expand environment variables. + +``kibana_discover_app_url: http://kibana:5601/#/discover`` + +kibana_discover_version +^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_version``: Specifies the version of the Kibana Discover application. + +The currently supported versions of Kibana Discover are: + +- `5.6` +- `6.0`, `6.1`, `6.2`, `6.3`, `6.4`, `6.5`, `6.6`, `6.7`, `6.8` +- `7.0`, `7.1`, `7.2`, `7.3` + +``kibana_discover_version: '7.3'`` + +kibana_discover_index_pattern_id +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_index_pattern_id``: The id of the index pattern to link to in the Kibana Discover application. +These ids are usually generated and can be found in url of the index pattern management page, or by exporting its saved object. + +Example export of an index pattern's saved object: + +.. code-block:: text + + [ + { + "_id": "4e97d188-8a45-4418-8a37-07ed69b4d34c", + "_type": "index-pattern", + "_source": { ... } + } + ] + +You can modify an index pattern's id by exporting the saved object, modifying the ``_id`` field, and re-importing. + +``kibana_discover_index_pattern_id: 4e97d188-8a45-4418-8a37-07ed69b4d34c`` + +kibana_discover_columns +^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_columns``: The columns to display in the generated Kibana Discover application link. +Defaults to the ``_source`` column. + +``kibana_discover_columns: [ timestamp, message ]`` + +kibana_discover_from_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_from_timedelta``: The offset to the `from` time of the Kibana Discover link's time range. +The `from` time is calculated by subtracting this timedelta from the event time. Defaults to 10 minutes. + +``kibana_discover_from_timedelta: minutes: 2`` + +kibana_discover_to_timedelta +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +``kibana_discover_to_timedelta``: The offset to the `to` time of the Kibana Discover link's time range. +The `to` time is calculated by adding this timedelta to the event time. Defaults to 10 minutes. + +``kibana_discover_to_timedelta: minutes: 2`` + use_local_time ^^^^^^^^^^^^^^ @@ -687,6 +787,8 @@ guaranteed to have the exact same results as with Elasticsearch. For example, an ``--alert``: Trigger real alerts instead of the debug (logging text) alert. +``--formatted-output``: Output results in formatted JSON. + .. note:: Results from running this script may not always be the same as if an actual ElastAlert instance was running. Some rule types, such as spike and flatline require a minimum elapsed time before they begin alerting, based on their timeframe. In addition, use_count_query and @@ -785,7 +887,7 @@ may be counted on a per-``query_key`` basis. This rule requires two additional options: -``num_events``: The number of events which will trigger an alert. +``num_events``: The number of events which will trigger an alert, inclusive. ``timeframe``: The time that ``num_events`` must occur within. @@ -1069,6 +1171,9 @@ Optional: ``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and evaluated separately against the threshold(s). +``min_doc_count``: The minimum number of events in the current window needed for an alert to trigger. Used in conjunction with ``query_key``, +this will only consider terms which in their last ``buffer_time`` had at least ``min_doc_count`` records. Default 1. + ``use_run_every_query_size``: By default the metric value is calculated over a ``buffer_time`` sized window. If this parameter is true the rule will use ``run_every`` as the calculation window. @@ -1088,6 +1193,54 @@ allign with the time elastalert runs, (This both avoid calculations on partial d See: https://www.elastic.co/guide/en/elasticsearch/reference/current/search-aggregations-bucket-datehistogram-aggregation.html#_offset for a more comprehensive explaination. +Spike Aggregation +~~~~~~~~~~~~~~~~~~ + +``spike_aggregation``: This rule matches when the value of a metric within the calculation window is ``spike_height`` times larger or smaller +than during the previous time period. It uses two sliding windows to compare the current and reference metric values. +We will call these two windows "reference" and "current". + +This rule requires: + +``metric_agg_key``: This is the name of the field over which the metric value will be calculated. The underlying type of this field must be +supported by the specified aggregation type. If using a scripted field via ``metric_agg_script``, this is the name for your scripted field + +``metric_agg_type``: The type of metric aggregation to perform on the ``metric_agg_key`` field. This must be one of 'min', 'max', 'avg', +'sum', 'cardinality', 'value_count'. + +``spike_height``: The ratio of the metric value in the last ``timeframe`` to the previous ``timeframe`` that when hit +will trigger an alert. + +``spike_type``: Either 'up', 'down' or 'both'. 'Up' meaning the rule will only match when the metric value is ``spike_height`` times +higher. 'Down' meaning the reference metric value is ``spike_height`` higher than the current metric value. 'Both' will match either. + +``buffer_time``: The rule will average out the rate of events over this time period. For example, ``hours: 1`` means that the 'current' +window will span from present to one hour ago, and the 'reference' window will span from one hour ago to two hours ago. The rule +will not be active until the time elapsed from the first event is at least two timeframes. This is to prevent an alert being triggered +before a baseline rate has been established. This can be overridden using ``alert_on_new_data``. + +Optional: + +``query_key``: Group metric calculations by this field. For each unique value of the ``query_key`` field, the metric will be calculated and +evaluated separately against the 'reference'/'current' metric value and ``spike height``. + +``metric_agg_script``: A `Painless` formatted script describing how to calculate your metric on-the-fly:: + + metric_agg_key: myScriptedMetric + metric_agg_script: + script: doc['field1'].value * doc['field2'].value + +``threshold_ref``: The minimum value of the metric in the reference window for an alert to trigger. For example, if +``spike_height: 3`` and ``threshold_ref: 10``, then the 'reference' window must have a metric value of 10 and the 'current' window at +least three times that for an alert to be triggered. + +``threshold_cur``: The minimum value of the metric in the current window for an alert to trigger. For example, if +``spike_height: 3`` and ``threshold_cur: 60``, then an alert will occur if the current window has a metric value greater than 60 and +the reference window is less than a third of that value. + +``min_doc_count``: The minimum number of events in the current window needed for an alert to trigger. Used in conjunction with ``query_key``, +this will only consider terms which in their last ``buffer_time`` had at least ``min_doc_count`` records. Default 1. + Percentage Match ~~~~~~~~~~~~~~~~ @@ -1124,6 +1277,8 @@ evaluated separately against the threshold(s). For example, "%.2f" will round it to 2 decimal places. See: https://docs.python.org/3.4/library/string.html#format-specification-mini-language +``min_denominator``: Minimum number of documents on which percentage calculation will apply. Default is 0. + .. _alerts: Alerts @@ -1243,13 +1398,15 @@ With ``alert_text_type: aggregation_summary_only``:: body = rule_name aggregation_summary -+ + ruletype_text is the string returned by RuleType.get_match_str. field_values will contain every key value pair included in the results from Elasticsearch. These fields include "@timestamp" (or the value of ``timestamp_field``), every key in ``include``, every key in ``top_count_keys``, ``query_key``, and ``compare_key``. If the alert spans multiple events, these values may come from an individual event, usually the one which triggers the alert. +When using ``alert_text_args``, you can access nested fields and index into arrays. For example, if your match was ``{"data": {"ips": ["127.0.0.1", "12.34.56.78"]}}``, then by using ``"data.ips[1]"`` in ``alert_text_args``, it would replace value with ``"12.34.56.78"``. This can go arbitrarily deep into fields and will still work on keys that contain dots themselves. + Command ~~~~~~~ @@ -1474,9 +1631,11 @@ Optional: ``opsgenie_account``: The OpsGenie account to integrate with. ``opsgenie_recipients``: A list OpsGenie recipients who will be notified by the alert. - +``opsgenie_recipients_args``: Map of arguments used to format opsgenie_recipients. +``opsgenie_default_recipients``: List of default recipients to notify when the formatting of opsgenie_recipients is unsuccesful. ``opsgenie_teams``: A list of OpsGenie teams to notify (useful for schedules with escalation). - +``opsgenie_teams_args``: Map of arguments used to format opsgenie_teams (useful for assigning the alerts to teams based on some data) +``opsgenie_default_teams``: List of default teams to notify when the formatting of opsgenie_teams is unsuccesful. ``opsgenie_tags``: A list of tags for this alert. ``opsgenie_message``: Set the OpsGenie message to something other than the rule name. The message can be formatted with fields from the first match e.g. "Error occurred for {app_name} at {timestamp}.". @@ -1489,6 +1648,15 @@ Optional: ``opsgenie_priority``: Set the OpsGenie priority level. Possible values are P1, P2, P3, P4, P5. +``opsgenie_details``: Map of custom key/value pairs to include in the alert's details. The value can sourced from either fields in the first match, environment variables, or a constant value. + +Example usage:: + + opsgenie_details: + Author: 'Bob Smith' # constant value + Environment: '$VAR' # environment variable + Message: { field: message } # field in the first match + SNS ~~~ @@ -1616,6 +1784,47 @@ Provide absolute address of the pciture, for example: http://some.address.com/im ``slack_alert_fields``: You can add additional fields to your slack alerts using this field. Specify the title using `title` and a value for the field using `value`. Additionally you can specify whether or not this field should be a `short` field using `short: true`. +``slack_title``: Sets a title for the message, this shows up as a blue text at the start of the message + +``slack_title_link``: You can add a link in your Slack notification by setting this to a valid URL. Requires slack_title to be set. + +``slack_timeout``: You can specify a timeout value, in seconds, for making communicating with Slac. The default is 10. If a timeout occurs, the alert will be retried next time elastalert cycles. + +``slack_attach_kibana_discover_url``: Enables the attachment of the ``kibana_discover_url`` to the slack notification. The config ``generate_kibana_discover_url`` must also be ``True`` in order to generate the url. Defaults to ``False``. + +``slack_kibana_discover_color``: The color of the Kibana Discover url attachment. Defaults to ``#ec4b98``. + +``slack_kibana_discover_title``: The title of the Kibana Discover url attachment. Defaults to ``Discover in Kibana``. + +Mattermost +~~~~~~~~~~ + +Mattermost alerter will send a notification to a predefined Mattermost channel. The body of the notification is formatted the same as with other alerters. + +The alerter requires the following option: + +``mattermost_webhook_url``: The webhook URL. Follow the instructions on https://docs.mattermost.com/developer/webhooks-incoming.html to create an incoming webhook on your Mattermost installation. + +Optional: + +``mattermost_proxy``: By default ElastAlert will not use a network proxy to send notifications to Mattermost. Set this option using ``hostname:port`` if you need to use a proxy. + +``mattermost_ignore_ssl_errors``: By default ElastAlert will verify SSL certificate. Set this option to ``False`` if you want to ignore SSL errors. + +``mattermost_username_override``: By default Mattermost will use your username when posting to the channel. Use this option to change it (free text). + +``mattermost_channel_override``: Incoming webhooks have a default channel, but it can be overridden. A public channel can be specified "#other-channel", and a Direct Message with "@username". + +``mattermost_icon_url_override``: By default ElastAlert will use the default webhook icon when posting to the channel. You can provide icon_url to use custom image. +Provide absolute address of the picture (for example: http://some.address.com/image.jpg) or Base64 data url. + +``mattermost_msg_pretext``: You can set the message attachment pretext using this option. + +``mattermost_msg_color``: By default the alert will be posted with the 'danger' color. You can also use 'good', 'warning', or hex color code. + +``mattermost_msg_fields``: You can add fields to your Mattermost alerts using this option. You can specify the title using `title` and the text value using `value`. Additionally you can specify whether this field should be a `short` field using `short: true`. If you set `args` and `value` is a formattable string, ElastAlert will format the incident key based on the provided array of fields from the rule or match. +See https://docs.mattermost.com/developer/message-attachments.html#fields for more information. + Telegram ~~~~~~~~ @@ -1633,6 +1842,27 @@ Optional: ``telegram_proxy``: By default ElastAlert will not use a network proxy to send notifications to Telegram. Set this option using ``hostname:port`` if you need to use a proxy. +GoogleChat +~~~~~~~~~~ +GoogleChat alerter will send a notification to a predefined GoogleChat channel. The body of the notification is formatted the same as with other alerters. + +The alerter requires the following options: + +``googlechat_webhook_url``: The webhook URL that includes the channel (room) you want to post to. Go to the Google Chat website https://chat.google.com and choose the channel in which you wish to receive the notifications. Select 'Configure Webhooks' to create a new webhook or to copy the URL from an existing one. You can use a list of URLs to send to multiple channels. + +Optional: + +``googlechat_format``: Formatting for the notification. Can be either 'card' or 'basic' (default). + +``googlechat_header_title``: Sets the text for the card header title. (Only used if format=card) + +``googlechat_header_subtitle``: Sets the text for the card header subtitle. (Only used if format=card) + +``googlechat_header_image``: URL for the card header icon. (Only used if format=card) + +``googlechat_footer_kibanalink``: URL to Kibana to include in the card footer. (Only used if format=card) + + PagerDuty ~~~~~~~~~ @@ -1669,14 +1899,31 @@ See https://v2.developer.pagerduty.com/docs/send-an-event-events-api-v2 ``pagerduty_v2_payload_class``: Sets the class of the payload. (the event type in PagerDuty) +``pagerduty_v2_payload_class_args``: If set, and ``pagerduty_v2_payload_class`` is a formattable string, Elastalert will format the class based on the provided array of fields from the rule or match. + ``pagerduty_v2_payload_component``: Sets the component of the payload. (what program/interface/etc the event came from) +``pagerduty_v2_payload_component_args``: If set, and ``pagerduty_v2_payload_component`` is a formattable string, Elastalert will format the component based on the provided array of fields from the rule or match. + ``pagerduty_v2_payload_group``: Sets the logical grouping (e.g. app-stack) +``pagerduty_v2_payload_group_args``: If set, and ``pagerduty_v2_payload_group`` is a formattable string, Elastalert will format the group based on the provided array of fields from the rule or match. + ``pagerduty_v2_payload_severity``: Sets the severity of the page. (defaults to `critical`, valid options: `critical`, `error`, `warning`, `info`) ``pagerduty_v2_payload_source``: Sets the source of the event, preferably the hostname or fqdn. +``pagerduty_v2_payload_source_args``: If set, and ``pagerduty_v2_payload_source`` is a formattable string, Elastalert will format the source based on the provided array of fields from the rule or match. + +PagerTree +~~~~~~~~~ + +PagerTree alerter will trigger an incident to a predefined PagerTree integration url. + +The alerter requires the following options: + +``pagertree_integration_url``: URL generated by PagerTree for the integration. + Exotel ~~~~~~ @@ -1684,7 +1931,7 @@ Developers in India can use Exotel alerter, it will trigger an incident to a mob The alerter requires the following option: -``exotel_accout_sid``: This is sid of your Exotel account. +``exotel_account_sid``: This is sid of your Exotel account. ``exotel_auth_token``: Auth token assosiated with your Exotel account. @@ -1730,7 +1977,7 @@ The alerter requires the following options: Optional: -``victorops_entity_id``: The identity of the incident used by VictorOps to correlate incidents thoughout the alert lifecycle. If not defined, VictorOps will assign a random string to each alert. +``victorops_entity_id``: The identity of the incident used by VictorOps to correlate incidents throughout the alert lifecycle. If not defined, VictorOps will assign a random string to each alert. ``victorops_entity_display_name``: Human-readable name of alerting entity to summarize incidents without affecting the life-cycle workflow. @@ -1810,8 +2057,7 @@ Alerta ~~~~~~ Alerta alerter will post an alert in the Alerta server instance through the alert API endpoint. -The default values will work with a local Alerta server installation with authorization disabled. -See http://alerta.readthedocs.io/en/latest/api/alert.html for more details on the Alerta alert json format. +See http://alerta.readthedocs.io/en/latest/api/alert.html for more details on the Alerta JSON format. For Alerta 5.0 @@ -1821,46 +2067,47 @@ Required: Optional: -``alerta_api_key``: This is the api key for alerta server if required. Default behaviour is that no Authorization header sent with the request. +``alerta_api_key``: This is the api key for alerta server, sent in an ``Authorization`` HTTP header. If not defined, no Authorization header is sent. + +``alerta_use_qk_as_resource``: If true and query_key is present, this will override ``alerta_resource`` field with the ``query_key value`` (Can be useful if ``query_key`` is a hostname). -``alerta_resource``: The resource name of the generated alert. Defaults to "elastalert". Can be a reference to a part of the match. +``alerta_use_match_timestamp``: If true, it will use the timestamp of the first match as the ``createTime`` of the alert. otherwise, the current server time is used. -``alerta_service``: A list of service tags for the generated alert. Defaults to "elastalert". Can be a reference to a part of the match. +``alert_missing_value``: Text to replace any match field not found when formating strings. Defaults to ````. -``alerta_severity``: The severity level of the alert. Defaults to "warning". +The following options dictate the values of the API JSON payload: -``alerta_origin``: The origin field for the generated alert. Defaults to "elastalert". Can be a reference to a part of the match. +``alerta_severity``: Defaults to "warning". -``alerta_environment``: The environment field for the generated alert. Defaults to "Production". Can be a reference to a part of the match. +``alerta_timeout``: Defaults 84600 (1 Day). -``alerta_group``: The group field for the generated alert. No Default. Can be a reference to a part of the match. +``alerta_type``: Defaults to "elastalert". -``alerta_timeout``: The time in seconds before this alert will expire (in Alerta). Default 84600 (1 Day). +The following options use Python-like string syntax ``{}`` or ``%()s`` to access parts of the match, similar to the CommandAlerter. Ie: "Alert for {clientip}". +If the referenced key is not found in the match, it is replaced by the text indicated by the option ``alert_missing_value``. -``alerta_correlate``: A list of alerta events that this one correlates with. Default is an empty list. Can make reference to a part of the match to build the event name. +``alerta_resource``: Defaults to "elastalert". -``alerta_tags``: A list of alerta tags. Default is an empty list. Can be a reference to a part of the match. +``alerta_service``: Defaults to "elastalert". -``alerta_use_qk_as_resource``: If true and query_key is present this will override alerta_resource field with the query key value (Can be useful if query_key is a hostname). +``alerta_origin``: Defaults to "elastalert". -``alerta_use_match_timestamp``: If true will use the timestamp of the first match as the createTime of the alert, otherwise the current time is used. Default False. +``alerta_environment``: Defaults to "Production". -``alerta_event``: Can make reference to parts of the match to build the event name. Defaults to "elastalert". +``alerta_group``: Defaults to "". -``alerta_text``: Python-style string can be used to make reference to parts of the match. Defaults to "elastalert". +``alerta_correlate``: Defaults to an empty list. -``alerta_type``: Defaults to "elastalert". +``alerta_tags``: Defaults to an empty list. -``alerta_value``: Can be a reference to a part of the match. No Default. +``alerta_event``: Defaults to the rule's name. -``alerta_attributes_keys``: List of key names for the Alerta Attributes dictionary +``alerta_text``: Defaults to the rule's text according to its type. -``alerta_attributes_values``: List of values for the Alerta Attributes dictionary, corresponding in order to the described keys. Can be a reference to a part of the match. +``alerta_value``: Defaults to "". -.. info:: +The ``attributes`` dictionary is built by joining the lists from ``alerta_attributes_keys`` and ``alerta_attributes_values``, considered in order. - The optional values use Python-like string syntax ``{}`` or ``%()s`` to access parts of the match, similar to the CommandAlerter. Ie: "Alert for {clientip}" - If the referenced value is not found in the match, it is replaced by ```` or the text indicated by the rule in ``alert_missing_value``. Example usage using old-style format:: @@ -1874,12 +2121,19 @@ Example usage using old-style format:: alerta_text: "Probe %(hostname)s is UP at %(logdate)s GMT" alerta_value: "UP" +Example usage using new-style format:: + + alert: + - alerta + alerta_attributes_values: ["{key}", "{logdate}", "{sender_ip}" ] + alerta_text: "Probe {hostname} is UP at {logdate} GMT" + HTTP POST ~~~~~~~~~ -This alert type will send results to a JSON endpoint using HTTP POST. The key names are configurable so this is compatible with almost any endpoint. By default, the JSON will contain al the items from the match, unless you specify http_post_payload, in which case it will only contain those items. +This alert type will send results to a JSON endpoint using HTTP POST. The key names are configurable so this is compatible with almost any endpoint. By default, the JSON will contain all the items from the match, unless you specify http_post_payload, in which case it will only contain those items. Required: @@ -1891,6 +2145,8 @@ Optional: ``http_post_static_payload``: Key:value pairs of static parameters to be sent, along with the Elasticsearch results. Put your authentication or other information here. +``http_post_headers``: Key:value pairs of headers to be sent as part of the request. + ``http_post_proxy``: URL of proxy, if required. ``http_post_all_values``: Boolean of whether or not to include every key value pair from the match in addition to those in http_post_payload and http_post_static_payload. Defaults to True if http_post_payload is not specified, otherwise False. @@ -1905,6 +2161,8 @@ Example usage:: ip: clientip http_post_static_payload: apikey: abc123 + http_post_headers: + authorization: Basic 123dr3234 Alerter @@ -1916,3 +2174,72 @@ Example usage:: jira_priority: $priority$ jira_alert_owner: $owner$ + + + +Line Notify +~~~~~~~~~~~ + +Line Notify will send notification to a Line application. The body of the notification is formatted the same as with other alerters. + +Required: + +``linenotify_access_token``: The access token that you got from https://notify-bot.line.me/my/ + +theHive +~~~~~~~ + +theHive alert type will send JSON request to theHive (Security Incident Response Platform) with TheHive4py API. Sent request will be stored like Hive Alert with description and observables. + +Required: + +``hive_connection``: The connection details as key:values. Required keys are ``hive_host``, ``hive_port`` and ``hive_apikey``. + +``hive_alert_config``: Configuration options for the alert. + +Optional: + +``hive_proxies``: Proxy configuration. + +``hive_observable_data_mapping``: If needed, matched data fields can be mapped to TheHive observable types using python string formatting. + +Example usage:: + + alert: hivealerter + + hive_connection: + hive_host: http://localhost + hive_port: + hive_apikey: + hive_proxies: + http: '' + https: '' + + hive_alert_config: + title: 'Title' ## This will default to {rule[index]_rule[name]} if not provided + type: 'external' + source: 'elastalert' + description: '{match[field1]} {rule[name]} Sample description' + severity: 2 + tags: ['tag1', 'tag2 {rule[name]}'] + tlp: 3 + status: 'New' + follow: True + + hive_observable_data_mapping: + - domain: "{match[field1]}_{rule[name]}" + - domain: "{match[field]}" + - ip: "{match[ip_field]}" + + +Zabbix +~~~~~~~~~~~ + +Zabbix will send notification to a Zabbix server. The item in the host specified receive a 1 value for each hit. For example, if the elastic query produce 3 hits in the last execution of elastalert, three '1' (integer) values will be send from elastalert to Zabbix Server. If the query have 0 hits, any value will be sent. + +Required: + +``zbx_sender_host``: The address where zabbix server is running. +``zbx_sender_port``: The port where zabbix server is listenning. +``zbx_host``: This field setup the host in zabbix that receives the value sent by Elastalert. +``zbx_item``: This field setup the item in the host that receives the value sent by Elastalert. diff --git a/docs/source/running_elastalert.rst b/docs/source/running_elastalert.rst index 09e307c24..7fdf1eeba 100644 --- a/docs/source/running_elastalert.rst +++ b/docs/source/running_elastalert.rst @@ -8,7 +8,7 @@ Requirements - Elasticsearch - ISO8601 or Unix timestamped data -- Python 2.7 +- Python 3.6 - pip, see requirements.txt - Packages on Ubuntu 14.x: python-pip python-dev libffi-dev libssl-dev diff --git a/elastalert/__init__.py b/elastalert/__init__.py index e69de29bb..55bfdb32f 100644 --- a/elastalert/__init__.py +++ b/elastalert/__init__.py @@ -0,0 +1,257 @@ +# -*- coding: utf-8 -*- +import copy +import time + +from elasticsearch import Elasticsearch +from elasticsearch import RequestsHttpConnection +from elasticsearch.client import _make_path +from elasticsearch.client import query_params +from elasticsearch.exceptions import TransportError + + +class ElasticSearchClient(Elasticsearch): + """ Extension of low level :class:`Elasticsearch` client with additional version resolving features """ + + def __init__(self, conf): + """ + :arg conf: es_conn_config dictionary. Ref. :func:`~util.build_es_conn_config` + """ + super(ElasticSearchClient, self).__init__(host=conf['es_host'], + port=conf['es_port'], + url_prefix=conf['es_url_prefix'], + use_ssl=conf['use_ssl'], + verify_certs=conf['verify_certs'], + ca_certs=conf['ca_certs'], + connection_class=RequestsHttpConnection, + http_auth=conf['http_auth'], + timeout=conf['es_conn_timeout'], + send_get_body_as=conf['send_get_body_as'], + client_cert=conf['client_cert'], + client_key=conf['client_key']) + self._conf = copy.copy(conf) + self._es_version = None + + @property + def conf(self): + """ + Returns the provided es_conn_config used when initializing the class instance. + """ + return self._conf + + @property + def es_version(self): + """ + Returns the reported version from the Elasticsearch server. + """ + if self._es_version is None: + for retry in range(3): + try: + self._es_version = self.info()['version']['number'] + break + except TransportError: + if retry == 2: + raise + time.sleep(3) + return self._es_version + + def is_atleastfive(self): + """ + Returns True when the Elasticsearch server version >= 5 + """ + return int(self.es_version.split(".")[0]) >= 5 + + def is_atleastsix(self): + """ + Returns True when the Elasticsearch server version >= 6 + """ + return int(self.es_version.split(".")[0]) >= 6 + + def is_atleastsixtwo(self): + """ + Returns True when the Elasticsearch server version >= 6.2 + """ + major, minor = list(map(int, self.es_version.split(".")[:2])) + return major > 6 or (major == 6 and minor >= 2) + + def is_atleastsixsix(self): + """ + Returns True when the Elasticsearch server version >= 6.6 + """ + major, minor = list(map(int, self.es_version.split(".")[:2])) + return major > 6 or (major == 6 and minor >= 6) + + def is_atleastseven(self): + """ + Returns True when the Elasticsearch server version >= 7 + """ + return int(self.es_version.split(".")[0]) >= 7 + + def resolve_writeback_index(self, writeback_index, doc_type): + """ In ES6, you cannot have multiple _types per index, + therefore we use self.writeback_index as the prefix for the actual + index name, based on doc_type. """ + if not self.is_atleastsix(): + return writeback_index + elif doc_type == 'silence': + return writeback_index + '_silence' + elif doc_type == 'past_elastalert': + return writeback_index + '_past' + elif doc_type == 'elastalert_status': + return writeback_index + '_status' + elif doc_type == 'elastalert_error': + return writeback_index + '_error' + return writeback_index + + @query_params( + "_source", + "_source_exclude", + "_source_excludes", + "_source_include", + "_source_includes", + "allow_no_indices", + "allow_partial_search_results", + "analyze_wildcard", + "analyzer", + "batched_reduce_size", + "default_operator", + "df", + "docvalue_fields", + "expand_wildcards", + "explain", + "from_", + "ignore_unavailable", + "lenient", + "max_concurrent_shard_requests", + "pre_filter_shard_size", + "preference", + "q", + "rest_total_hits_as_int", + "request_cache", + "routing", + "scroll", + "search_type", + "seq_no_primary_term", + "size", + "sort", + "stats", + "stored_fields", + "suggest_field", + "suggest_mode", + "suggest_size", + "suggest_text", + "terminate_after", + "timeout", + "track_scores", + "track_total_hits", + "typed_keys", + "version", + ) + def deprecated_search(self, index=None, doc_type=None, body=None, params=None): + """ + Execute a search query and get back search hits that match the query. + ``_ + :arg index: A list of index names to search, or a string containing a + comma-separated list of index names to search; use `_all` + or empty string to perform the operation on all indices + :arg doc_type: A comma-separated list of document types to search; leave + empty to perform the operation on all types + :arg body: The search definition using the Query DSL + :arg _source: True or false to return the _source field or not, or a + list of fields to return + :arg _source_exclude: A list of fields to exclude from the returned + _source field + :arg _source_include: A list of fields to extract and return from the + _source field + :arg allow_no_indices: Whether to ignore if a wildcard indices + expression resolves into no concrete indices. (This includes `_all` + string or when no indices have been specified) + :arg allow_partial_search_results: Set to false to return an overall + failure if the request would produce partial results. Defaults to + True, which will allow partial results in the case of timeouts or + partial failures + :arg analyze_wildcard: Specify whether wildcard and prefix queries + should be analyzed (default: false) + :arg analyzer: The analyzer to use for the query string + :arg batched_reduce_size: The number of shard results that should be + reduced at once on the coordinating node. This value should be used + as a protection mechanism to reduce the memory overhead per search + request if the potential number of shards in the request can be + large., default 512 + :arg default_operator: The default operator for query string query (AND + or OR), default 'OR', valid choices are: 'AND', 'OR' + :arg df: The field to use as default where no field prefix is given in + the query string + :arg docvalue_fields: A comma-separated list of fields to return as the + docvalue representation of a field for each hit + :arg expand_wildcards: Whether to expand wildcard expression to concrete + indices that are open, closed or both., default 'open', valid + choices are: 'open', 'closed', 'none', 'all' + :arg explain: Specify whether to return detailed information about score + computation as part of a hit + :arg from\\_: Starting offset (default: 0) + :arg ignore_unavailable: Whether specified concrete indices should be + ignored when unavailable (missing or closed) + :arg lenient: Specify whether format-based query failures (such as + providing text to a numeric field) should be ignored + :arg max_concurrent_shard_requests: The number of concurrent shard + requests this search executes concurrently. This value should be + used to limit the impact of the search on the cluster in order to + limit the number of concurrent shard requests, default 'The default + grows with the number of nodes in the cluster but is at most 256.' + :arg pre_filter_shard_size: A threshold that enforces a pre-filter + roundtrip to prefilter search shards based on query rewriting if + the number of shards the search request expands to exceeds the + threshold. This filter roundtrip can limit the number of shards + significantly if for instance a shard can not match any documents + based on it's rewrite method ie. if date filters are mandatory to + match but the shard bounds and the query are disjoint., default 128 + :arg preference: Specify the node or shard the operation should be + performed on (default: random) + :arg q: Query in the Lucene query string syntax + :arg rest_total_hits_as_int: This parameter is used to restore the total hits as a number + in the response. This param is added version 6.x to handle mixed cluster queries where nodes + are in multiple versions (7.0 and 6.latest) + :arg request_cache: Specify if request cache should be used for this + request or not, defaults to index level setting + :arg routing: A comma-separated list of specific routing values + :arg scroll: Specify how long a consistent view of the index should be + maintained for scrolled search + :arg search_type: Search operation type, valid choices are: + 'query_then_fetch', 'dfs_query_then_fetch' + :arg size: Number of hits to return (default: 10) + :arg sort: A comma-separated list of : pairs + :arg stats: Specific 'tag' of the request for logging and statistical + purposes + :arg stored_fields: A comma-separated list of stored fields to return as + part of a hit + :arg suggest_field: Specify which field to use for suggestions + :arg suggest_mode: Specify suggest mode, default 'missing', valid + choices are: 'missing', 'popular', 'always' + :arg suggest_size: How many suggestions to return in response + :arg suggest_text: The source text for which the suggestions should be + returned + :arg terminate_after: The maximum number of documents to collect for + each shard, upon reaching which the query execution will terminate + early. + :arg timeout: Explicit operation timeout + :arg track_scores: Whether to calculate and return scores even if they + are not used for sorting + :arg track_total_hits: Indicate if the number of documents that match + the query should be tracked + :arg typed_keys: Specify whether aggregation and suggester names should + be prefixed by their respective types in the response + :arg version: Specify whether to return document version as part of a + hit + """ + # from is a reserved word so it cannot be used, use from_ instead + if "from_" in params: + params["from"] = params.pop("from_") + + if not index: + index = "_all" + res = self.transport.perform_request( + "GET", _make_path(index, doc_type, "_search"), params=params, body=body + ) + if type(res) == list or type(res) == tuple: + return res[1] + return res diff --git a/elastalert/alerts.py b/elastalert/alerts.py index 653091fb0..f2f31853f 100644 --- a/elastalert/alerts.py +++ b/elastalert/alerts.py @@ -4,13 +4,15 @@ import json import logging import os +import re import subprocess import sys import time +import uuid import warnings from email.mime.text import MIMEText from email.utils import formatdate -from HTMLParser import HTMLParser +from html.parser import HTMLParser from smtplib import SMTP from smtplib import SMTP_SSL from smtplib import SMTPAuthenticationError @@ -23,18 +25,20 @@ from exotel import Exotel from jira.client import JIRA from jira.exceptions import JIRAError +from requests.auth import HTTPProxyAuth from requests.exceptions import RequestException from staticconf.loader import yaml_loader from texttable import Texttable from twilio.base.exceptions import TwilioRestException from twilio.rest import Client as TwilioClient -from util import EAException -from util import elastalert_logger -from util import lookup_es_key -from util import pretty_ts -from util import resolve_string -from util import ts_now -from util import ts_to_dt + +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key +from .util import pretty_ts +from .util import resolve_string +from .util import ts_now +from .util import ts_to_dt class DateTimeEncoder(json.JSONEncoder): @@ -58,7 +62,7 @@ def _ensure_new_line(self): def _add_custom_alert_text(self): missing = self.rule.get('alert_missing_value', '') - alert_text = unicode(self.rule.get('alert_text', '')) + alert_text = str(self.rule.get('alert_text', '')) if 'alert_text_args' in self.rule: alert_text_args = self.rule.get('alert_text_args') alert_text_values = [lookup_es_key(self.match, arg) for arg in alert_text_args] @@ -76,7 +80,7 @@ def _add_custom_alert_text(self): alert_text = alert_text.format(*alert_text_values) elif 'alert_text_kw' in self.rule: kw = {} - for name, kw_name in self.rule.get('alert_text_kw').items(): + for name, kw_name in list(self.rule.get('alert_text_kw').items()): val = lookup_es_key(self.match, name) # Support referencing other top-level rule properties @@ -94,10 +98,10 @@ def _add_rule_text(self): self.text += self.rule['type'].get_match_str(self.match) def _add_top_counts(self): - for key, counts in self.match.items(): + for key, counts in list(self.match.items()): if key.startswith('top_events_'): self.text += '%s:\n' % (key[11:]) - top_events = counts.items() + top_events = list(counts.items()) if not top_events: self.text += 'No events found.\n' @@ -109,12 +113,12 @@ def _add_top_counts(self): self.text += '\n' def _add_match_items(self): - match_items = self.match.items() + match_items = list(self.match.items()) match_items.sort(key=lambda x: x[0]) for key, value in match_items: if key.startswith('top_events_'): continue - value_str = unicode(value) + value_str = str(value) value_str.replace('\\n', '\n') if type(value) in [list, dict]: try: @@ -150,9 +154,9 @@ def __str__(self): class JiraFormattedMatchString(BasicMatchString): def _add_match_items(self): - match_items = dict([(x, y) for x, y in self.match.items() if not x.startswith('top_events_')]) + match_items = dict([(x, y) for x, y in list(self.match.items()) if not x.startswith('top_events_')]) json_blob = self._pretty_print_as_json(match_items) - preformatted_text = u'{{code}}{0}{{code}}'.format(json_blob) + preformatted_text = '{{code}}{0}{{code}}'.format(json_blob) self.text += preformatted_text @@ -181,14 +185,14 @@ def resolve_rule_references(self, root): root[i] = self.resolve_rule_reference(item) elif type(root) == dict: # Make a copy since we may be modifying the contents of the structure we're walking - for key, value in root.copy().iteritems(): + for key, value in root.copy().items(): if type(value) == dict or type(value) == list: self.resolve_rule_references(root[key]) else: root[key] = self.resolve_rule_reference(value) def resolve_rule_reference(self, value): - strValue = unicode(value) + strValue = str(value) if strValue.startswith('$') and strValue.endswith('$') and strValue[1:-1] in self.rule: if type(value) == int: return int(self.rule[strValue[1:-1]]) @@ -220,7 +224,8 @@ def create_title(self, matches): return self.create_default_title(matches) def create_custom_title(self, matches): - alert_subject = unicode(self.rule['alert_subject']) + alert_subject = str(self.rule['alert_subject']) + alert_subject_max_len = int(self.rule.get('alert_subject_max_len', 2048)) if 'alert_subject_args' in self.rule: alert_subject_args = self.rule['alert_subject_args'] @@ -237,7 +242,10 @@ def create_custom_title(self, matches): missing = self.rule.get('alert_missing_value', '') alert_subject_values = [missing if val is None else val for val in alert_subject_values] - return alert_subject.format(*alert_subject_values) + alert_subject = alert_subject.format(*alert_subject_values) + + if len(alert_subject) > alert_subject_max_len: + alert_subject = alert_subject[:alert_subject_max_len] return alert_subject @@ -245,7 +253,7 @@ def create_alert_body(self, matches): body = self.get_aggregation_summary_text(matches) if self.rule.get('alert_text_type') != 'aggregation_summary_only': for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' @@ -258,6 +266,7 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = '' if 'aggregation' in self.rule and 'summary_table_fields' in self.rule: + text = self.rule.get('summary_prefix', '') summary_table_fields = self.rule['summary_table_fields'] if not isinstance(summary_table_fields, list): summary_table_fields = [summary_table_fields] @@ -274,16 +283,16 @@ def get_aggregation_summary_text(self, matches): # Maintain an aggregate count for each unique key encountered in the aggregation period for match in matches: - key_tuple = tuple([unicode(lookup_es_key(match, key)) for key in summary_table_fields]) + key_tuple = tuple([str(lookup_es_key(match, key)) for key in summary_table_fields]) if key_tuple not in match_aggregation: match_aggregation[key_tuple] = 1 else: match_aggregation[key_tuple] = match_aggregation[key_tuple] + 1 - for keys, count in match_aggregation.iteritems(): + for keys, count in match_aggregation.items(): text_table.add_row([key for key in keys] + [count]) text += text_table.draw() + '\n\n' - - return unicode(text) + text += self.rule.get('summary_prefix', '') + return str(text) def create_default_title(self, matches): return self.rule['name'] @@ -339,13 +348,13 @@ def alert(self, matches): ) fullmessage['match'] = lookup_es_key( match, self.rule['timestamp_field']) - elastalert_logger.info(unicode(BasicMatchString(self.rule, match))) + elastalert_logger.info(str(BasicMatchString(self.rule, match))) fullmessage['alerts'] = alerts fullmessage['rule'] = self.rule['name'] fullmessage['rule_file'] = self.rule['rule_file'] - fullmessage['matching'] = unicode(BasicMatchString(self.rule, match)) + fullmessage['matching'] = str(BasicMatchString(self.rule, match)) fullmessage['alertDate'] = datetime.datetime.now( ).strftime("%Y-%m-%d %H:%M:%S") fullmessage['body'] = self.create_alert_body(matches) @@ -358,8 +367,9 @@ def alert(self, matches): self.stomp_password = self.rule.get('stomp_password', 'admin') self.stomp_destination = self.rule.get( 'stomp_destination', '/queue/ALERT') + self.stomp_ssl = self.rule.get('stomp_ssl', False) - conn = stomp.Connection([(self.stomp_hostname, self.stomp_hostport)]) + conn = stomp.Connection([(self.stomp_hostname, self.stomp_hostport)], use_ssl=self.stomp_ssl) conn.start() conn.connect(self.stomp_login, self.stomp_password) @@ -383,7 +393,7 @@ def alert(self, matches): 'Alert for %s, %s at %s:' % (self.rule['name'], match[qk], lookup_es_key(match, self.rule['timestamp_field']))) else: elastalert_logger.info('Alert for %s at %s:' % (self.rule['name'], lookup_es_key(match, self.rule['timestamp_field']))) - elastalert_logger.info(unicode(BasicMatchString(self.rule, match))) + elastalert_logger.info(str(BasicMatchString(self.rule, match))) def get_info(self): return {'type': 'debug'} @@ -405,15 +415,15 @@ def __init__(self, *args): self.smtp_key_file = self.rule.get('smtp_key_file') self.smtp_cert_file = self.rule.get('smtp_cert_file') # Convert email to a list if it isn't already - if isinstance(self.rule['email'], basestring): + if isinstance(self.rule['email'], str): self.rule['email'] = [self.rule['email']] # If there is a cc then also convert it a list if it isn't cc = self.rule.get('cc') - if cc and isinstance(cc, basestring): + if cc and isinstance(cc, str): self.rule['cc'] = [self.rule['cc']] # If there is a bcc then also convert it to a list if it isn't bcc = self.rule.get('bcc') - if bcc and isinstance(bcc, basestring): + if bcc and isinstance(bcc, str): self.rule['bcc'] = [self.rule['bcc']] add_suffix = self.rule.get('email_add_domain') if add_suffix and not add_suffix.startswith('@'): @@ -430,7 +440,7 @@ def alert(self, matches): to_addr = self.rule['email'] if 'email_from_field' in self.rule: recipient = lookup_es_key(matches[0], self.rule['email_from_field']) - if isinstance(recipient, basestring): + if isinstance(recipient, str): if '@' in recipient: to_addr = [recipient] elif 'email_add_domain' in self.rule: @@ -440,9 +450,9 @@ def alert(self, matches): if 'email_add_domain' in self.rule: to_addr = [name + self.rule['email_add_domain'] for name in to_addr] if self.rule.get('email_format') == 'html': - email_msg = MIMEText(body.encode('UTF-8'), 'html', _charset='UTF-8') + email_msg = MIMEText(body, 'html', _charset='UTF-8') else: - email_msg = MIMEText(body.encode('UTF-8'), _charset='UTF-8') + email_msg = MIMEText(body, _charset='UTF-8') email_msg['Subject'] = self.create_title(matches) email_msg['To'] = ', '.join(to_addr) email_msg['From'] = self.from_addr @@ -475,7 +485,7 @@ def alert(self, matches): except SMTPAuthenticationError as e: raise EAException("SMTP username/password rejected: %s" % (e)) self.smtp.sendmail(self.from_addr, to_addr, email_msg.as_string()) - self.smtp.close() + self.smtp.quit() elastalert_logger.info("Sent email to %s" % (to_addr)) @@ -587,7 +597,7 @@ def __init__(self, rule): self.get_arbitrary_fields() except JIRAError as e: # JIRAError may contain HTML, pass along only first 1024 chars - raise EAException("Error connecting to JIRA: %s" % (str(e)[:1024])), None, sys.exc_info()[2] + raise EAException("Error connecting to JIRA: %s" % (str(e)[:1024])).with_traceback(sys.exc_info()[2]) self.set_priority() @@ -596,7 +606,7 @@ def set_priority(self): if self.priority is not None and self.client is not None: self.jira_args['priority'] = {'id': self.priority_ids[self.priority]} except KeyError: - logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, self.priority_ids.keys())) + logging.error("Priority %s not found. Valid priorities are %s" % (self.priority, list(self.priority_ids.keys()))) def reset_jira_args(self): self.jira_args = {'project': {'key': self.project}, @@ -690,7 +700,7 @@ def get_arbitrary_fields(self): # Clear jira_args self.reset_jira_args() - for jira_field, value in self.rule.iteritems(): + for jira_field, value in self.rule.items(): # If we find a field that is not covered by the set that we are aware of, it means it is either: # 1. A built-in supported field in JIRA that we don't have on our radar # 2. A custom field that a JIRA admin has configured @@ -746,7 +756,7 @@ def find_existing_ticket(self, matches): return issues[0] def comment_on_ticket(self, ticket, match): - text = unicode(JiraFormattedMatchString(self.rule, match)) + text = str(JiraFormattedMatchString(self.rule, match)) timestamp = pretty_ts(lookup_es_key(match, self.rule['timestamp_field'])) comment = "This alert was triggered again at %s\n%s" % (timestamp, text) self.client.add_comment(ticket, comment) @@ -784,9 +794,9 @@ def alert(self, matches): except JIRAError as e: logging.exception("Error while commenting on ticket %s: %s" % (ticket, e)) if self.labels: - for l in self.labels: + for label in self.labels: try: - ticket.fields.labels.append(l) + ticket.fields.labels.append(label) except JIRAError as e: logging.exception("Error while appending labels to ticket %s: %s" % (ticket, e)) if self.transition: @@ -821,7 +831,7 @@ def alert(self, matches): "Exception encountered when trying to add '{0}' as a watcher. Does the user exist?\n{1}" .format( watcher, ex - )), None, sys.exc_info()[2] + )).with_traceback(sys.exc_info()[2]) except JIRAError as e: raise EAException("Error creating JIRA ticket using jira_args (%s): %s" % (self.jira_args, e)) @@ -836,7 +846,7 @@ def create_alert_body(self, matches): body += self.get_aggregation_summary_text(matches) if self.rule.get('alert_text_type') != 'aggregation_summary_only': for match in matches: - body += unicode(JiraFormattedMatchString(self.rule, match)) + body += str(JiraFormattedMatchString(self.rule, match)) if len(matches) > 1: body += '\n----------------------------------------\n' return body @@ -844,7 +854,7 @@ def create_alert_body(self, matches): def get_aggregation_summary_text(self, matches): text = super(JiraAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'{{noformat}}{0}{{noformat}}'.format(text) + text = '{{noformat}}{0}{{noformat}}'.format(text) return text def create_default_title(self, matches, for_search=False): @@ -858,7 +868,9 @@ def create_default_title(self, matches, for_search=False): if for_search: return title - title += ' - %s' % (pretty_ts(matches[0][self.rule['timestamp_field']], self.rule.get('use_local_time'))) + timestamp = matches[0].get(self.rule['timestamp_field']) + if timestamp: + title += ' - %s' % (pretty_ts(timestamp, self.rule.get('use_local_time'))) # Add count for spikes count = matches[0].get('spike_count') @@ -880,7 +892,7 @@ def __init__(self, *args): self.last_command = [] self.shell = False - if isinstance(self.rule['command'], basestring): + if isinstance(self.rule['command'], str): self.shell = True if '%' in self.rule['command']: logging.warning('Warning! You could be vulnerable to shell injection!') @@ -904,10 +916,10 @@ def alert(self, matches): if self.rule.get('pipe_match_json'): match_json = json.dumps(matches, cls=DateTimeEncoder) + '\n' - stdout, stderr = subp.communicate(input=match_json) + stdout, stderr = subp.communicate(input=match_json.encode()) elif self.rule.get('pipe_alert_text'): alert_text = self.create_alert_body(matches) - stdout, stderr = subp.communicate(input=alert_text) + stdout, stderr = subp.communicate(input=alert_text.encode()) if self.rule.get("fail_on_non_zero_exit", False) and subp.wait(): raise EAException("Non-zero exit code while running command %s" % (' '.join(command))) except OSError as e: @@ -1046,7 +1058,7 @@ class MsTeamsAlerter(Alerter): def __init__(self, rule): super(MsTeamsAlerter, self).__init__(rule) self.ms_teams_webhook_url = self.rule['ms_teams_webhook_url'] - if isinstance(self.ms_teams_webhook_url, basestring): + if isinstance(self.ms_teams_webhook_url, str): self.ms_teams_webhook_url = [self.ms_teams_webhook_url] self.ms_teams_proxy = self.rule.get('ms_teams_proxy', None) self.ms_teams_alert_summary = self.rule.get('ms_teams_alert_summary', 'ElastAlert Message') @@ -1054,7 +1066,6 @@ def __init__(self, rule): self.ms_teams_theme_color = self.rule.get('ms_teams_theme_color', '') def format_body(self, body): - body = body.encode('UTF-8') if self.ms_teams_alert_fixed_width: body = body.replace('`', "'") body = "```{0}```".format('```\n\n```'.join(x for x in body.split('\n'))).replace('\n``````', '') @@ -1098,11 +1109,15 @@ class SlackAlerter(Alerter): def __init__(self, rule): super(SlackAlerter, self).__init__(rule) self.slack_webhook_url = self.rule['slack_webhook_url'] - if isinstance(self.slack_webhook_url, basestring): + if isinstance(self.slack_webhook_url, str): self.slack_webhook_url = [self.slack_webhook_url] self.slack_proxy = self.rule.get('slack_proxy', None) self.slack_username_override = self.rule.get('slack_username_override', 'elastalert') self.slack_channel_override = self.rule.get('slack_channel_override', '') + if isinstance(self.slack_channel_override, str): + self.slack_channel_override = [self.slack_channel_override] + self.slack_title_link = self.rule.get('slack_title_link', '') + self.slack_title = self.rule.get('slack_title', '') self.slack_emoji_override = self.rule.get('slack_emoji_override', ':ghost:') self.slack_icon_url_override = self.rule.get('slack_icon_url_override', '') self.slack_msg_color = self.rule.get('slack_msg_color', 'danger') @@ -1110,10 +1125,15 @@ def __init__(self, rule): self.slack_text_string = self.rule.get('slack_text_string', '') self.slack_alert_fields = self.rule.get('slack_alert_fields', '') self.slack_ignore_ssl_errors = self.rule.get('slack_ignore_ssl_errors', False) + self.slack_timeout = self.rule.get('slack_timeout', 10) + self.slack_ca_certs = self.rule.get('slack_ca_certs') + self.slack_attach_kibana_discover_url = self.rule.get('slack_attach_kibana_discover_url', False) + self.slack_kibana_discover_color = self.rule.get('slack_kibana_discover_color', '#ec4b98') + self.slack_kibana_discover_title = self.rule.get('slack_kibana_discover_title', 'Discover in Kibana') def format_body(self, body): # https://api.slack.com/docs/formatting - return body.encode('UTF-8') + return body def get_aggregation_summary_text__maximum_width(self): width = super(SlackAlerter, self).get_aggregation_summary_text__maximum_width() @@ -1123,7 +1143,7 @@ def get_aggregation_summary_text__maximum_width(self): def get_aggregation_summary_text(self, matches): text = super(SlackAlerter, self).get_aggregation_summary_text(matches) if text: - text = u'```\n{0}```\n'.format(text) + text = '```\n{0}```\n'.format(text) return text def populate_fields(self, matches): @@ -1144,7 +1164,6 @@ def alert(self, matches): proxies = {'https': self.slack_proxy} if self.slack_proxy else None payload = { 'username': self.slack_username_override, - 'channel': self.slack_channel_override, 'parse': self.slack_parse_override, 'text': self.slack_text_string, 'attachments': [ @@ -1167,24 +1186,154 @@ def alert(self, matches): else: payload['icon_emoji'] = self.slack_emoji_override + if self.slack_title != '': + payload['attachments'][0]['title'] = self.slack_title + + if self.slack_title_link != '': + payload['attachments'][0]['title_link'] = self.slack_title_link + + if self.slack_attach_kibana_discover_url: + kibana_discover_url = lookup_es_key(matches[0], 'kibana_discover_url') + if kibana_discover_url: + payload['attachments'].append({ + 'color': self.slack_kibana_discover_color, + 'title': self.slack_kibana_discover_title, + 'title_link': kibana_discover_url + }) + for url in self.slack_webhook_url: + for channel_override in self.slack_channel_override: + try: + if self.slack_ca_certs: + verify = self.slack_ca_certs + else: + verify = self.slack_ignore_ssl_errors + if self.slack_ignore_ssl_errors: + requests.packages.urllib3.disable_warnings() + payload['channel'] = channel_override + response = requests.post( + url, data=json.dumps(payload, cls=DateTimeEncoder), + headers=headers, verify=verify, + proxies=proxies, + timeout=self.slack_timeout) + warnings.resetwarnings() + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to slack: %s" % e) + elastalert_logger.info("Alert '%s' sent to Slack" % self.rule['name']) + + def get_info(self): + return {'type': 'slack', + 'slack_username_override': self.slack_username_override} + + +class MattermostAlerter(Alerter): + """ Creates a Mattermsot post for each alert """ + required_options = frozenset(['mattermost_webhook_url']) + + def __init__(self, rule): + super(MattermostAlerter, self).__init__(rule) + + # HTTP config + self.mattermost_webhook_url = self.rule['mattermost_webhook_url'] + if isinstance(self.mattermost_webhook_url, str): + self.mattermost_webhook_url = [self.mattermost_webhook_url] + self.mattermost_proxy = self.rule.get('mattermost_proxy', None) + self.mattermost_ignore_ssl_errors = self.rule.get('mattermost_ignore_ssl_errors', False) + + # Override webhook config + self.mattermost_username_override = self.rule.get('mattermost_username_override', 'elastalert') + self.mattermost_channel_override = self.rule.get('mattermost_channel_override', '') + self.mattermost_icon_url_override = self.rule.get('mattermost_icon_url_override', '') + + # Message properties + self.mattermost_msg_pretext = self.rule.get('mattermost_msg_pretext', '') + self.mattermost_msg_color = self.rule.get('mattermost_msg_color', 'danger') + self.mattermost_msg_fields = self.rule.get('mattermost_msg_fields', '') + + def get_aggregation_summary_text__maximum_width(self): + width = super(MattermostAlerter, self).get_aggregation_summary_text__maximum_width() + # Reduced maximum width for prettier Mattermost display. + return min(width, 75) + + def get_aggregation_summary_text(self, matches): + text = super(MattermostAlerter, self).get_aggregation_summary_text(matches) + if text: + text = '```\n{0}```\n'.format(text) + return text + + def populate_fields(self, matches): + alert_fields = [] + missing = self.rule.get('alert_missing_value', '') + for field in self.mattermost_msg_fields: + field = copy.copy(field) + if 'args' in field: + args_values = [lookup_es_key(matches[0], arg) or missing for arg in field['args']] + if 'value' in field: + field['value'] = field['value'].format(*args_values) + else: + field['value'] = "\n".join(str(arg) for arg in args_values) + del(field['args']) + alert_fields.append(field) + return alert_fields + + def alert(self, matches): + body = self.create_alert_body(matches) + title = self.create_title(matches) + + # post to mattermost + headers = {'content-type': 'application/json'} + # set https proxy, if it was provided + proxies = {'https': self.mattermost_proxy} if self.mattermost_proxy else None + payload = { + 'attachments': [ + { + 'fallback': "{0}: {1}".format(title, self.mattermost_msg_pretext), + 'color': self.mattermost_msg_color, + 'title': title, + 'pretext': self.mattermost_msg_pretext, + 'fields': [] + } + ] + } + + if self.rule.get('alert_text_type') == 'alert_text_only': + payload['attachments'][0]['text'] = body + else: + payload['text'] = body + + if self.mattermost_msg_fields != '': + payload['attachments'][0]['fields'] = self.populate_fields(matches) + + if self.mattermost_icon_url_override != '': + payload['icon_url'] = self.mattermost_icon_url_override + + if self.mattermost_username_override != '': + payload['username'] = self.mattermost_username_override + + if self.mattermost_channel_override != '': + payload['channel'] = self.mattermost_channel_override + + for url in self.mattermost_webhook_url: try: - if self.slack_ignore_ssl_errors: - requests.packages.urllib3.disable_warnings() + if self.mattermost_ignore_ssl_errors: + requests.urllib3.disable_warnings() + response = requests.post( url, data=json.dumps(payload, cls=DateTimeEncoder), - headers=headers, verify=not self.slack_ignore_ssl_errors, + headers=headers, verify=not self.mattermost_ignore_ssl_errors, proxies=proxies) + warnings.resetwarnings() response.raise_for_status() except RequestException as e: - raise EAException("Error posting to slack: %s" % e) - elastalert_logger.info("Alert sent to Slack") + raise EAException("Error posting to Mattermost: %s" % e) + elastalert_logger.info("Alert sent to Mattermost") def get_info(self): - return {'type': 'slack', - 'slack_username_override': self.slack_username_override, - 'slack_webhook_url': self.slack_webhook_url} + return {'type': 'mattermost', + 'mattermost_username_override': self.mattermost_username_override, + 'mattermost_webhook_url': self.mattermost_webhook_url} class PagerDutyAlerter(Alerter): @@ -1202,10 +1351,14 @@ def __init__(self, rule): self.pagerduty_api_version = self.rule.get('pagerduty_api_version', 'v1') self.pagerduty_v2_payload_class = self.rule.get('pagerduty_v2_payload_class', '') + self.pagerduty_v2_payload_class_args = self.rule.get('pagerduty_v2_payload_class_args', None) self.pagerduty_v2_payload_component = self.rule.get('pagerduty_v2_payload_component', '') + self.pagerduty_v2_payload_component_args = self.rule.get('pagerduty_v2_payload_component_args', None) self.pagerduty_v2_payload_group = self.rule.get('pagerduty_v2_payload_group', '') + self.pagerduty_v2_payload_group_args = self.rule.get('pagerduty_v2_payload_group_args', None) self.pagerduty_v2_payload_severity = self.rule.get('pagerduty_v2_payload_severity', 'critical') self.pagerduty_v2_payload_source = self.rule.get('pagerduty_v2_payload_source', 'ElastAlert') + self.pagerduty_v2_payload_source_args = self.rule.get('pagerduty_v2_payload_source_args', None) if self.pagerduty_api_version == 'v2': self.url = 'https://events.pagerduty.com/v2/enqueue' @@ -1224,17 +1377,28 @@ def alert(self, matches): 'dedup_key': self.get_incident_key(matches), 'client': self.pagerduty_client_name, 'payload': { - 'class': self.pagerduty_v2_payload_class, - 'component': self.pagerduty_v2_payload_component, - 'group': self.pagerduty_v2_payload_group, + 'class': self.resolve_formatted_key(self.pagerduty_v2_payload_class, + self.pagerduty_v2_payload_class_args, + matches), + 'component': self.resolve_formatted_key(self.pagerduty_v2_payload_component, + self.pagerduty_v2_payload_component_args, + matches), + 'group': self.resolve_formatted_key(self.pagerduty_v2_payload_group, + self.pagerduty_v2_payload_group_args, + matches), 'severity': self.pagerduty_v2_payload_severity, - 'source': self.pagerduty_v2_payload_source, + 'source': self.resolve_formatted_key(self.pagerduty_v2_payload_source, + self.pagerduty_v2_payload_source_args, + matches), 'summary': self.create_title(matches), 'custom_details': { - 'information': body.encode('UTF-8'), + 'information': body, }, }, } + match_timestamp = lookup_es_key(matches[0], self.rule.get('timestamp_field', '@timestamp')) + if match_timestamp: + payload['payload']['timestamp'] = match_timestamp else: payload = { 'service_key': self.pagerduty_service_key, @@ -1243,7 +1407,7 @@ def alert(self, matches): 'incident_key': self.get_incident_key(matches), 'client': self.pagerduty_client_name, 'details': { - "information": body.encode('UTF-8'), + "information": body, }, } @@ -1267,6 +1431,23 @@ def alert(self, matches): elif self.pagerduty_event_type == 'acknowledge': elastalert_logger.info("acknowledge sent to PagerDuty") + def resolve_formatted_key(self, key, args, matches): + if args: + key_values = [lookup_es_key(matches[0], arg) for arg in args] + + # Populate values with rule level properties too + for i in range(len(key_values)): + if key_values[i] is None: + key_value = self.rule.get(args[i]) + if key_value: + key_values[i] = key_value + + missing = self.rule.get('alert_missing_value', '') + key_values = [missing if val is None else val for val in key_values] + return key.format(*key_values) + else: + return key + def get_incident_key(self, matches): if self.pagerduty_incident_key_args: incident_key_values = [lookup_es_key(matches[0], arg) for arg in self.pagerduty_incident_key_args] @@ -1289,6 +1470,39 @@ def get_info(self): 'pagerduty_client_name': self.pagerduty_client_name} +class PagerTreeAlerter(Alerter): + """ Creates a PagerTree Incident for each alert """ + required_options = frozenset(['pagertree_integration_url']) + + def __init__(self, rule): + super(PagerTreeAlerter, self).__init__(rule) + self.url = self.rule['pagertree_integration_url'] + self.pagertree_proxy = self.rule.get('pagertree_proxy', None) + + def alert(self, matches): + # post to pagertree + headers = {'content-type': 'application/json'} + # set https proxy, if it was provided + proxies = {'https': self.pagertree_proxy} if self.pagertree_proxy else None + payload = { + "event_type": "create", + "Id": str(uuid.uuid4()), + "Title": self.create_title(matches), + "Description": self.create_alert_body(matches) + } + + try: + response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies) + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to PagerTree: %s" % e) + elastalert_logger.info("Trigger sent to PagerTree") + + def get_info(self): + return {'type': 'pagertree', + 'pagertree_integration_url': self.url} + + class ExotelAlerter(Alerter): required_options = frozenset(['exotel_account_sid', 'exotel_auth_token', 'exotel_to_number', 'exotel_from_number']) @@ -1309,7 +1523,7 @@ def alert(self, matches): if response != 200: raise EAException("Error posting to Exotel, response code is %s" % response) except RequestException: - raise EAException("Error posting to Exotel"), None, sys.exc_info()[2] + raise EAException("Error posting to Exotel").with_traceback(sys.exc_info()[2]) elastalert_logger.info("Trigger sent to Exotel") def get_info(self): @@ -1398,21 +1612,24 @@ def __init__(self, rule): self.telegram_api_url = self.rule.get('telegram_api_url', 'api.telegram.org') self.url = 'https://%s/bot%s/%s' % (self.telegram_api_url, self.telegram_bot_token, "sendMessage") self.telegram_proxy = self.rule.get('telegram_proxy', None) + self.telegram_proxy_login = self.rule.get('telegram_proxy_login', None) + self.telegram_proxy_password = self.rule.get('telegram_proxy_pass', None) def alert(self, matches): - body = u'āš  *%s* āš  ```\n' % (self.create_title(matches)) + body = 'āš  *%s* āš  ```\n' % (self.create_title(matches)) for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' if len(body) > 4095: body = body[0:4000] + "\nāš  *message was cropped according to telegram limits!* āš " - body += u' ```' + body += ' ```' headers = {'content-type': 'application/json'} # set https proxy, if it was provided proxies = {'https': self.telegram_proxy} if self.telegram_proxy else None + auth = HTTPProxyAuth(self.telegram_proxy_login, self.telegram_proxy_password) if self.telegram_proxy_login else None payload = { 'chat_id': self.telegram_room_id, 'text': body, @@ -1421,7 +1638,7 @@ def alert(self, matches): } try: - response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies) + response = requests.post(self.url, data=json.dumps(payload, cls=DateTimeEncoder), headers=headers, proxies=proxies, auth=auth) warnings.resetwarnings() response.raise_for_status() except RequestException as e: @@ -1435,6 +1652,95 @@ def get_info(self): 'telegram_room_id': self.telegram_room_id} +class GoogleChatAlerter(Alerter): + """ Send a notification via Google Chat webhooks """ + required_options = frozenset(['googlechat_webhook_url']) + + def __init__(self, rule): + super(GoogleChatAlerter, self).__init__(rule) + self.googlechat_webhook_url = self.rule['googlechat_webhook_url'] + if isinstance(self.googlechat_webhook_url, str): + self.googlechat_webhook_url = [self.googlechat_webhook_url] + self.googlechat_format = self.rule.get('googlechat_format', 'basic') + self.googlechat_header_title = self.rule.get('googlechat_header_title', None) + self.googlechat_header_subtitle = self.rule.get('googlechat_header_subtitle', None) + self.googlechat_header_image = self.rule.get('googlechat_header_image', None) + self.googlechat_footer_kibanalink = self.rule.get('googlechat_footer_kibanalink', None) + + def create_header(self): + header = None + if self.googlechat_header_title: + header = { + "title": self.googlechat_header_title, + "subtitle": self.googlechat_header_subtitle, + "imageUrl": self.googlechat_header_image + } + return header + + def create_footer(self): + footer = None + if self.googlechat_footer_kibanalink: + footer = {"widgets": [{ + "buttons": [{ + "textButton": { + "text": "VISIT KIBANA", + "onClick": { + "openLink": { + "url": self.googlechat_footer_kibanalink + } + } + } + }] + }] + } + return footer + + def create_card(self, matches): + card = {"cards": [{ + "sections": [{ + "widgets": [ + {"textParagraph": {"text": self.create_alert_body(matches)}} + ]} + ]} + ]} + + # Add the optional header + header = self.create_header() + if header: + card['cards'][0]['header'] = header + + # Add the optional footer + footer = self.create_footer() + if footer: + card['cards'][0]['sections'].append(footer) + return card + + def create_basic(self, matches): + body = self.create_alert_body(matches) + return {'text': body} + + def alert(self, matches): + # Format message + if self.googlechat_format == 'card': + message = self.create_card(matches) + else: + message = self.create_basic(matches) + + # Post to webhook + headers = {'content-type': 'application/json'} + for url in self.googlechat_webhook_url: + try: + response = requests.post(url, data=json.dumps(message), headers=headers) + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to google chat: {}".format(e)) + elastalert_logger.info("Alert sent to Google Chat!") + + def get_info(self): + return {'type': 'googlechat', + 'googlechat_webhook_url': self.googlechat_webhook_url} + + class GitterAlerter(Alerter): """ Creates a Gitter activity message for each alert """ required_options = frozenset(['gitter_webhook_url']) @@ -1535,18 +1841,21 @@ class AlertaAlerter(Alerter): def __init__(self, rule): super(AlertaAlerter, self).__init__(rule) + # Setup defaul parameters self.url = self.rule.get('alerta_api_url', None) - - # Fill up default values self.api_key = self.rule.get('alerta_api_key', None) + self.timeout = self.rule.get('alerta_timeout', 86400) + self.use_match_timestamp = self.rule.get('alerta_use_match_timestamp', False) + self.use_qk_as_resource = self.rule.get('alerta_use_qk_as_resource', False) + self.verify_ssl = not self.rule.get('alerta_api_skip_ssl', False) + self.missing_text = self.rule.get('alert_missing_value', '') + + # Fill up default values of the API JSON payload self.severity = self.rule.get('alerta_severity', 'warning') self.resource = self.rule.get('alerta_resource', 'elastalert') self.environment = self.rule.get('alerta_environment', 'Production') self.origin = self.rule.get('alerta_origin', 'elastalert') self.service = self.rule.get('alerta_service', ['elastalert']) - self.timeout = self.rule.get('alerta_timeout', 86400) - self.use_match_timestamp = self.rule.get('alerta_use_match_timestamp', False) - self.use_qk_as_resource = self.rule.get('alerta_use_qk_as_resource', False) self.text = self.rule.get('alerta_text', 'elastalert') self.type = self.rule.get('alerta_type', 'elastalert') self.event = self.rule.get('alerta_event', 'elastalert') @@ -1557,8 +1866,6 @@ def __init__(self, rule): self.attributes_values = self.rule.get('alerta_attributes_values', []) self.value = self.rule.get('alerta_value', '') - self.missing_text = self.rule.get('alert_missing_value', '') - def alert(self, matches): # Override the resource if requested if self.use_qk_as_resource and 'query_key' in self.rule and lookup_es_key(matches[0], self.rule['query_key']): @@ -1567,12 +1874,10 @@ def alert(self, matches): headers = {'content-type': 'application/json'} if self.api_key is not None: headers['Authorization'] = 'Key %s' % (self.rule['alerta_api_key']) - alerta_payload = self.get_json_payload(matches[0]) try: - - response = requests.post(self.url, data=alerta_payload, headers=headers) + response = requests.post(self.url, data=alerta_payload, headers=headers, verify=self.verify_ssl) response.raise_for_status() except RequestException as e: raise EAException("Error posting to Alerta: %s" % e) @@ -1580,7 +1885,7 @@ def alert(self, matches): def create_default_title(self, matches): title = '%s' % (self.rule['name']) - # If the rule has a query_key, add that value plus timestamp to subject + # If the rule has a query_key, add that value if 'query_key' in self.rule: qk = matches[0].get(self.rule['query_key']) if qk: @@ -1600,17 +1905,11 @@ def get_json_payload(self, match): """ - alerta_service = [resolve_string(a_service, match, self.missing_text) for a_service in self.service] - alerta_tags = [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags] - alerta_correlate = [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate] - alerta_attributes_values = [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values] - alerta_text = resolve_string(self.text, match, self.missing_text) - alerta_text = self.rule['type'].get_match_str([match]) if alerta_text == '' else alerta_text - alerta_event = resolve_string(self.event, match, self.missing_text) - alerta_event = self.create_default_title([match]) if alerta_event == '' else alerta_event - - timestamp_field = self.rule.get('timestamp_field', '@timestamp') - match_timestamp = lookup_es_key(match, timestamp_field) + # Using default text and event title if not defined in rule + alerta_text = self.rule['type'].get_match_str([match]) if self.text == '' else resolve_string(self.text, match, self.missing_text) + alerta_event = self.create_default_title([match]) if self.event == '' else resolve_string(self.event, match, self.missing_text) + + match_timestamp = lookup_es_key(match, self.rule.get('timestamp_field', '@timestamp')) if match_timestamp is None: match_timestamp = datetime.datetime.now().strftime("%Y-%m-%dT%H:%M:%S.%fZ") if self.use_match_timestamp: @@ -1630,10 +1929,11 @@ def get_json_payload(self, match): 'event': alerta_event, 'text': alerta_text, 'value': resolve_string(self.value, match, self.missing_text), - 'service': alerta_service, - 'tags': alerta_tags, - 'correlate': alerta_correlate, - 'attributes': dict(zip(self.attributes_keys, alerta_attributes_values)), + 'service': [resolve_string(a_service, match, self.missing_text) for a_service in self.service], + 'tags': [resolve_string(a_tag, match, self.missing_text) for a_tag in self.tags], + 'correlate': [resolve_string(an_event, match, self.missing_text) for an_event in self.correlate], + 'attributes': dict(list(zip(self.attributes_keys, + [resolve_string(a_value, match, self.missing_text) for a_value in self.attributes_values]))), 'rawData': self.create_alert_body([match]), } @@ -1650,7 +1950,7 @@ class HTTPPostAlerter(Alerter): def __init__(self, rule): super(HTTPPostAlerter, self).__init__(rule) post_url = self.rule.get('http_post_url') - if isinstance(post_url, basestring): + if isinstance(post_url, str): post_url = [post_url] self.post_url = post_url self.post_proxy = self.rule.get('http_post_proxy') @@ -1665,7 +1965,7 @@ def alert(self, matches): for match in matches: payload = match if self.post_all_values else {} payload.update(self.post_static_payload) - for post_key, es_key in self.post_payload.items(): + for post_key, es_key in list(self.post_payload.items()): payload[post_key] = lookup_es_key(match, es_key) headers = { "Content-Type": "application/json", @@ -1777,4 +2077,110 @@ def alert(self, matches): def get_info(self): return {'type': 'stride', 'stride_cloud_id': self.stride_cloud_id, - 'stride_conversation_id': self.stride_conversation_id} + 'stride_converstation_id': self.stride_converstation_id} + + +class LineNotifyAlerter(Alerter): + """ Created a Line Notify for each alert """ + required_option = frozenset(["linenotify_access_token"]) + + def __init__(self, rule): + super(LineNotifyAlerter, self).__init__(rule) + self.linenotify_access_token = self.rule["linenotify_access_token"] + + def alert(self, matches): + body = self.create_alert_body(matches) + # post to Line Notify + headers = { + "Content-Type": "application/x-www-form-urlencoded", + "Authorization": "Bearer {}".format(self.linenotify_access_token) + } + payload = { + "message": body + } + try: + response = requests.post("https://notify-api.line.me/api/notify", data=payload, headers=headers) + response.raise_for_status() + except RequestException as e: + raise EAException("Error posting to Line Notify: %s" % e) + elastalert_logger.info("Alert sent to Line Notify") + + def get_info(self): + return {"type": "linenotify", "linenotify_access_token": self.linenotify_access_token} + + +class HiveAlerter(Alerter): + """ + Use matched data to create alerts containing observables in an instance of TheHive + """ + + required_options = set(['hive_connection', 'hive_alert_config']) + + def alert(self, matches): + + connection_details = self.rule['hive_connection'] + + for match in matches: + context = {'rule': self.rule, 'match': match} + + artifacts = [] + for mapping in self.rule.get('hive_observable_data_mapping', []): + for observable_type, match_data_key in mapping.items(): + try: + match_data_keys = re.findall(r'\{match\[([^\]]*)\]', match_data_key) + rule_data_keys = re.findall(r'\{rule\[([^\]]*)\]', match_data_key) + data_keys = match_data_keys + rule_data_keys + context_keys = list(context['match'].keys()) + list(context['rule'].keys()) + if all([True if k in context_keys else False for k in data_keys]): + artifact = {'tlp': 2, 'tags': [], 'message': None, 'dataType': observable_type, + 'data': match_data_key.format(**context)} + artifacts.append(artifact) + except KeyError: + raise KeyError('\nformat string\n{}\nmatch data\n{}'.format(match_data_key, context)) + + alert_config = { + 'artifacts': artifacts, + 'sourceRef': str(uuid.uuid4())[0:6], + 'customFields': {}, + 'caseTemplate': None, + 'title': '{rule[index]}_{rule[name]}'.format(**context), + 'date': int(time.time()) * 1000 + } + alert_config.update(self.rule.get('hive_alert_config', {})) + custom_fields = {} + for alert_config_field, alert_config_value in alert_config.items(): + if alert_config_field == 'customFields': + n = 0 + for cf_key, cf_value in alert_config_value.items(): + cf = {'order': n, cf_value['type']: cf_value['value'].format(**context)} + n += 1 + custom_fields[cf_key] = cf + elif isinstance(alert_config_value, str): + alert_config[alert_config_field] = alert_config_value.format(**context) + elif isinstance(alert_config_value, (list, tuple)): + formatted_list = [] + for element in alert_config_value: + try: + formatted_list.append(element.format(**context)) + except (AttributeError, KeyError, IndexError): + formatted_list.append(element) + alert_config[alert_config_field] = formatted_list + if custom_fields: + alert_config['customFields'] = custom_fields + + alert_body = json.dumps(alert_config, indent=4, sort_keys=True) + req = '{}:{}/api/alert'.format(connection_details['hive_host'], connection_details['hive_port']) + headers = {'Content-Type': 'application/json', 'Authorization': 'Bearer {}'.format(connection_details.get('hive_apikey', ''))} + proxies = connection_details.get('hive_proxies', {'http': '', 'https': ''}) + verify = connection_details.get('hive_verify', False) + response = requests.post(req, headers=headers, data=alert_body, proxies=proxies, verify=verify) + + if response.status_code != 201: + raise Exception('alert not successfully created in TheHive\n{}'.format(response.text)) + + def get_info(self): + + return { + 'type': 'hivealerter', + 'hive_host': self.rule.get('hive_connection', {}).get('hive_host', '') + } diff --git a/elastalert/config.py b/elastalert/config.py index c6efb3ad2..5ae9a26e6 100644 --- a/elastalert/config.py +++ b/elastalert/config.py @@ -1,36 +1,18 @@ # -*- coding: utf-8 -*- -import copy import datetime -import hashlib import logging -import os -import sys +import logging.config -import alerts -import enhancements -import jsonschema -import ruletypes -import yaml -import yaml.scanner from envparse import Env -from opsgenie import OpsGenieAlerter from staticconf.loader import yaml_loader -from util import dt_to_ts -from util import dt_to_ts_with_format -from util import dt_to_unix -from util import dt_to_unixms -from util import EAException -from util import ts_to_dt -from util import ts_to_dt_with_format -from util import unix_to_dt -from util import unixms_to_dt -# schema for rule yaml -rule_schema = jsonschema.Draft4Validator(yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')))) +from . import loaders +from .util import EAException +from .util import elastalert_logger +from .util import get_module -# Required global (config.yaml) and local (rule.yaml) configuration options -required_globals = frozenset(['run_every', 'rules_folder', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) -required_locals = frozenset(['alert', 'type', 'name', 'index']) +# Required global (config.yaml) configuration options +required_globals = frozenset(['run_every', 'es_host', 'es_port', 'writeback_index', 'buffer_time']) # Settings that can be derived from ENV variables env_settings = {'ES_USE_SSL': 'use_ssl', @@ -42,415 +24,57 @@ env = Env(ES_USE_SSL=bool) -# import rule dependency -import_rules = {} -# Used to map the names of rules to their classes -rules_mapping = { - 'frequency': ruletypes.FrequencyRule, - 'any': ruletypes.AnyRule, - 'spike': ruletypes.SpikeRule, - 'blacklist': ruletypes.BlacklistRule, - 'whitelist': ruletypes.WhitelistRule, - 'change': ruletypes.ChangeRule, - 'flatline': ruletypes.FlatlineRule, - 'new_term': ruletypes.NewTermsRule, - 'cardinality': ruletypes.CardinalityRule, - 'metric_aggregation': ruletypes.MetricAggregationRule, - 'percentage_match': ruletypes.PercentageMatchRule, +# Used to map the names of rule loaders to their classes +loader_mapping = { + 'file': loaders.FileRulesLoader, } -# Used to map names of alerts to their classes -alerts_mapping = { - 'email': alerts.EmailAlerter, - 'jira': alerts.JiraAlerter, - 'opsgenie': OpsGenieAlerter, - 'stomp': alerts.StompAlerter, - 'debug': alerts.DebugAlerter, - 'command': alerts.CommandAlerter, - 'sns': alerts.SnsAlerter, - 'hipchat': alerts.HipChatAlerter, - 'stride': alerts.StrideAlerter, - 'ms_teams': alerts.MsTeamsAlerter, - 'slack': alerts.SlackAlerter, - 'pagerduty': alerts.PagerDutyAlerter, - 'exotel': alerts.ExotelAlerter, - 'twilio': alerts.TwilioAlerter, - 'victorops': alerts.VictorOpsAlerter, - 'telegram': alerts.TelegramAlerter, - 'gitter': alerts.GitterAlerter, - 'servicenow': alerts.ServiceNowAlerter, - 'alerta': alerts.AlertaAlerter, - 'post': alerts.HTTPPostAlerter -} -# A partial ordering of alert types. Relative order will be preserved in the resulting alerts list -# For example, jira goes before email so the ticket # will be added to the resulting email. -alerts_order = { - 'jira': 0, - 'email': 1 -} - -base_config = {} - - -def get_module(module_name): - """ Loads a module and returns a specific object. - module_name should 'module.file.object'. - Returns object or raises EAException on error. """ - try: - module_path, module_class = module_name.rsplit('.', 1) - base_module = __import__(module_path, globals(), locals(), [module_class]) - module = getattr(base_module, module_class) - except (ImportError, AttributeError, ValueError) as e: - raise EAException("Could not import module %s: %s" % (module_name, e)), None, sys.exc_info()[2] - return module - - -def load_configuration(filename, conf, args=None): - """ Load a yaml rule file and fill in the relevant fields with objects. - - :param filename: The name of a rule configuration file. - :param conf: The global configuration dictionary, used for populating defaults. - :return: The rule configuration, a dictionary. - """ - rule = load_rule_yaml(filename) - load_options(rule, conf, filename, args) - load_modules(rule, args) - return rule - - -def load_rule_yaml(filename): - rule = { - 'rule_file': filename, - } - - import_rules.pop(filename, None) # clear `filename` dependency - while True: - try: - loaded = yaml_loader(filename) - except yaml.scanner.ScannerError as e: - raise EAException('Could not parse file %s: %s' % (filename, e)) - - # Special case for merging filters - if both files specify a filter merge (AND) them - if 'filter' in rule and 'filter' in loaded: - rule['filter'] = loaded['filter'] + rule['filter'] - - loaded.update(rule) - rule = loaded - if 'import' in rule: - # Find the path of the next file. - if os.path.isabs(rule['import']): - import_filename = rule['import'] - else: - import_filename = os.path.join(os.path.dirname(filename), rule['import']) - # set dependencies - rules = import_rules.get(filename, []) - rules.append(import_filename) - import_rules[filename] = rules - filename = import_filename - del(rule['import']) # or we could go on forever! - else: - break - - return rule - - -def load_options(rule, conf, filename, args=None): - """ Converts time objects, sets defaults, and validates some settings. - - :param rule: A dictionary of parsed YAML from a rule config file. - :param conf: The global configuration dictionary, used for populating defaults. - """ - adjust_deprecated_values(rule) - - try: - rule_schema.validate(rule) - except jsonschema.ValidationError as e: - raise EAException("Invalid Rule file: %s\n%s" % (filename, e)) - - try: - # Set all time based parameters - if 'timeframe' in rule: - rule['timeframe'] = datetime.timedelta(**rule['timeframe']) - if 'realert' in rule: - rule['realert'] = datetime.timedelta(**rule['realert']) - else: - if 'aggregation' in rule: - rule['realert'] = datetime.timedelta(minutes=0) - else: - rule['realert'] = datetime.timedelta(minutes=1) - if 'aggregation' in rule and not rule['aggregation'].get('schedule'): - rule['aggregation'] = datetime.timedelta(**rule['aggregation']) - if 'query_delay' in rule: - rule['query_delay'] = datetime.timedelta(**rule['query_delay']) - if 'buffer_time' in rule: - rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) - if 'bucket_interval' in rule: - rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) - if 'exponential_realert' in rule: - rule['exponential_realert'] = datetime.timedelta(**rule['exponential_realert']) - if 'kibana4_start_timedelta' in rule: - rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) - if 'kibana4_end_timedelta' in rule: - rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) - except (KeyError, TypeError) as e: - raise EAException('Invalid time format used: %s' % (e)) - - # Set defaults, copy defaults from config.yaml - for key, val in base_config.items(): - rule.setdefault(key, val) - rule.setdefault('name', os.path.splitext(filename)[0]) - rule.setdefault('realert', datetime.timedelta(seconds=0)) - rule.setdefault('aggregation', datetime.timedelta(seconds=0)) - rule.setdefault('query_delay', datetime.timedelta(seconds=0)) - rule.setdefault('timestamp_field', '@timestamp') - rule.setdefault('filter', []) - rule.setdefault('timestamp_type', 'iso') - rule.setdefault('timestamp_format', '%Y-%m-%dT%H:%M:%SZ') - rule.setdefault('_source_enabled', True) - rule.setdefault('use_local_time', True) - rule.setdefault('description', "") - - # Set timestamp_type conversion function, used when generating queries and processing hits - rule['timestamp_type'] = rule['timestamp_type'].strip().lower() - if rule['timestamp_type'] == 'iso': - rule['ts_to_dt'] = ts_to_dt - rule['dt_to_ts'] = dt_to_ts - elif rule['timestamp_type'] == 'unix': - rule['ts_to_dt'] = unix_to_dt - rule['dt_to_ts'] = dt_to_unix - elif rule['timestamp_type'] == 'unix_ms': - rule['ts_to_dt'] = unixms_to_dt - rule['dt_to_ts'] = dt_to_unixms - elif rule['timestamp_type'] == 'custom': - def _ts_to_dt_with_format(ts): - return ts_to_dt_with_format(ts, ts_format=rule['timestamp_format']) - - def _dt_to_ts_with_format(dt): - ts = dt_to_ts_with_format(dt, ts_format=rule['timestamp_format']) - if 'timestamp_format_expr' in rule: - # eval expression passing 'ts' and 'dt' - return eval(rule['timestamp_format_expr'], {'ts': ts, 'dt': dt}) - else: - return ts - - rule['ts_to_dt'] = _ts_to_dt_with_format - rule['dt_to_ts'] = _dt_to_ts_with_format - else: - raise EAException('timestamp_type must be one of iso, unix, or unix_ms') - - # Add support for client ssl certificate auth - if 'verify_certs' in conf: - rule.setdefault('verify_certs', conf.get('verify_certs')) - rule.setdefault('ca_certs', conf.get('ca_certs')) - rule.setdefault('client_cert', conf.get('client_cert')) - rule.setdefault('client_key', conf.get('client_key')) - - # Set HipChat options from global config - rule.setdefault('hipchat_msg_color', 'red') - rule.setdefault('hipchat_domain', 'api.hipchat.com') - rule.setdefault('hipchat_notify', True) - rule.setdefault('hipchat_from', '') - rule.setdefault('hipchat_ignore_ssl_errors', False) - - # Make sure we have required options - if required_locals - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(required_locals - frozenset(rule.keys())))) - - if 'include' in rule and type(rule['include']) != list: - raise EAException('include option must be a list') - - if isinstance(rule.get('query_key'), list): - rule['compound_query_key'] = rule['query_key'] - rule['query_key'] = ','.join(rule['query_key']) - - if isinstance(rule.get('aggregation_key'), list): - rule['compound_aggregation_key'] = rule['aggregation_key'] - rule['aggregation_key'] = ','.join(rule['aggregation_key']) - - if isinstance(rule.get('compare_key'), list): - rule['compound_compare_key'] = rule['compare_key'] - rule['compare_key'] = ','.join(rule['compare_key']) - elif 'compare_key' in rule: - rule['compound_compare_key'] = [rule['compare_key']] - # Add QK, CK and timestamp to include - include = rule.get('include', ['*']) - if 'query_key' in rule: - include.append(rule['query_key']) - if 'compound_query_key' in rule: - include += rule['compound_query_key'] - if 'compound_aggregation_key' in rule: - include += rule['compound_aggregation_key'] - if 'compare_key' in rule: - include.append(rule['compare_key']) - if 'compound_compare_key' in rule: - include += rule['compound_compare_key'] - if 'top_count_keys' in rule: - include += rule['top_count_keys'] - include.append(rule['timestamp_field']) - rule['include'] = list(set(include)) - - # Check that generate_kibana_url is compatible with the filters - if rule.get('generate_kibana_link'): - for es_filter in rule.get('filter'): - if es_filter: - if 'not' in es_filter: - es_filter = es_filter['not'] - if 'query' in es_filter: - es_filter = es_filter['query'] - if es_filter.keys()[0] not in ('term', 'query_string', 'range'): - raise EAException('generate_kibana_link is incompatible with filters other than term, query_string and range. ' - 'Consider creating a dashboard and using use_kibana_dashboard instead.') - - # Check that doc_type is provided if use_count/terms_query - if rule.get('use_count_query') or rule.get('use_terms_query'): - if 'doc_type' not in rule: - raise EAException('doc_type must be specified.') - # Check that query_key is set if use_terms_query - if rule.get('use_terms_query'): - if 'query_key' not in rule: - raise EAException('query_key must be specified with use_terms_query') - - # Warn if use_strf_index is used with %y, %M or %D - # (%y = short year, %M = minutes, %D = full date) - if rule.get('use_strftime_index'): - for token in ['%y', '%M', '%D']: - if token in rule.get('index'): - logging.warning('Did you mean to use %s in the index? ' - 'The index will be formatted like %s' % (token, - datetime.datetime.now().strftime(rule.get('index')))) - - if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): - raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') - - -def load_modules(rule, args=None): - """ Loads things that could be modules. Enhancements, alerts and rule type. """ - # Set match enhancements - match_enhancements = [] - for enhancement_name in rule.get('match_enhancements', []): - if enhancement_name in dir(enhancements): - enhancement = getattr(enhancements, enhancement_name) - else: - enhancement = get_module(enhancement_name) - if not issubclass(enhancement, enhancements.BaseEnhancement): - raise EAException("Enhancement module %s not a subclass of BaseEnhancement" % (enhancement_name)) - match_enhancements.append(enhancement(rule)) - rule['match_enhancements'] = match_enhancements - - # Convert rule type into RuleType object - if rule['type'] in rules_mapping: - rule['type'] = rules_mapping[rule['type']] - else: - rule['type'] = get_module(rule['type']) - if not issubclass(rule['type'], ruletypes.RuleType): - raise EAException('Rule module %s is not a subclass of RuleType' % (rule['type'])) - - # Make sure we have required alert and type options - reqs = rule['type'].required_options - - if reqs - frozenset(rule.keys()): - raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(rule.keys())))) - # Instantiate rule - try: - rule['type'] = rule['type'](rule, args) - except (KeyError, EAException) as e: - raise EAException('Error initializing rule %s: %s' % (rule['name'], e)), None, sys.exc_info()[2] - # Instantiate alerts only if we're not in debug mode - # In debug mode alerts are not actually sent so don't bother instantiating them - if not args or not args.debug: - rule['alert'] = load_alerts(rule, alert_field=rule['alert']) - - -def isyaml(filename): - return filename.endswith('.yaml') or filename.endswith('.yml') - - -def get_file_paths(conf, use_rule=None): - # Passing a filename directly can bypass rules_folder and .yaml checks - if use_rule and os.path.isfile(use_rule): - return [use_rule] - rule_folder = conf['rules_folder'] - rule_files = [] - if conf['scan_subdirectories']: - for root, folders, files in os.walk(rule_folder): - for filename in files: - if use_rule and use_rule != filename: - continue - if isyaml(filename): - rule_files.append(os.path.join(root, filename)) - else: - for filename in os.listdir(rule_folder): - fullpath = os.path.join(rule_folder, filename) - if os.path.isfile(fullpath) and isyaml(filename): - rule_files.append(fullpath) - return rule_files - - -def load_alerts(rule, alert_field): - def normalize_config(alert): - """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. - This function normalizes them both to the latter format. """ - if isinstance(alert, basestring): - return alert, rule - elif isinstance(alert, dict): - name, config = iter(alert.items()).next() - config_copy = copy.copy(rule) - config_copy.update(config) # warning, this (intentionally) mutates the rule dict - return name, config_copy - else: - raise EAException() - - def create_alert(alert, alert_config): - alert_class = alerts_mapping.get(alert) or get_module(alert) - if not issubclass(alert_class, alerts.Alerter): - raise EAException('Alert module %s is not a subclass of Alerter' % (alert)) - missing_options = (rule['type'].required_options | alert_class.required_options) - frozenset(alert_config or []) - if missing_options: - raise EAException('Missing required option(s): %s' % (', '.join(missing_options))) - return alert_class(alert_config) - - try: - if type(alert_field) != list: - alert_field = [alert_field] - - alert_field = [normalize_config(x) for x in alert_field] - alert_field = sorted(alert_field, key=lambda (a, b): alerts_order.get(a, 1)) - # Convert all alerts into Alerter objects - alert_field = [create_alert(a, b) for a, b in alert_field] - - except (KeyError, EAException) as e: - raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)), None, sys.exc_info()[2] - - return alert_field - - -def load_rules(args): +def load_conf(args, defaults=None, overwrites=None): """ Creates a conf dictionary for ElastAlerter. Loads the global - config file and then each rule found in rules_folder. + config file and then each rule found in rules_folder. - :param args: The parsed arguments to ElastAlert - :return: The global configuration, a dictionary. - """ - names = [] + :param args: The parsed arguments to ElastAlert + :param defaults: Dictionary of default conf values + :param overwrites: Dictionary of conf values to override + :return: The global configuration, a dictionary. + """ filename = args.config - conf = yaml_loader(filename) - use_rule = args.rule + if filename: + conf = yaml_loader(filename) + else: + try: + conf = yaml_loader('config.yaml') + except FileNotFoundError: + raise EAException('No --config or config.yaml found') - for env_var, conf_var in env_settings.items(): + # init logging from config and set log levels according to command line options + configure_logging(args, conf) + + for env_var, conf_var in list(env_settings.items()): val = env(env_var, None) if val is not None: conf[conf_var] = val + for key, value in (iter(defaults.items()) if defaults is not None else []): + if key not in conf: + conf[key] = value + + for key, value in (iter(overwrites.items()) if overwrites is not None else []): + conf[key] = value + # Make sure we have all required globals - if required_globals - frozenset(conf.keys()): - raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(conf.keys())))) + if required_globals - frozenset(list(conf.keys())): + raise EAException('%s must contain %s' % (filename, ', '.join(required_globals - frozenset(list(conf.keys()))))) + conf.setdefault('writeback_alias', 'elastalert_alerts') conf.setdefault('max_query_size', 10000) conf.setdefault('scroll_keepalive', '30s') + conf.setdefault('max_scrolling_count', 0) conf.setdefault('disable_rules_on_error', True) conf.setdefault('scan_subdirectories', True) + conf.setdefault('rules_loader', 'file') # Convert run_every, buffer_time into a timedelta object try: @@ -465,56 +89,48 @@ def load_rules(args): else: conf['old_query_limit'] = datetime.timedelta(weeks=1) except (KeyError, TypeError) as e: - raise EAException('Invalid time format used: %s' % (e)) - - global base_config - base_config = copy.deepcopy(conf) + raise EAException('Invalid time format used: %s' % e) - # Load each rule configuration file - rules = [] - rule_files = get_file_paths(conf, use_rule) - for rule_file in rule_files: - try: - rule = load_configuration(rule_file, conf, args) - # By setting "is_enabled: False" in rule file, a rule is easily disabled - if 'is_enabled' in rule and not rule['is_enabled']: - continue - if rule['name'] in names: - raise EAException('Duplicate rule named %s' % (rule['name'])) - except EAException as e: - raise EAException('Error loading file %s: %s' % (rule_file, e)) - - rules.append(rule) - names.append(rule['name']) + # Initialise the rule loader and load each rule configuration + rules_loader_class = loader_mapping.get(conf['rules_loader']) or get_module(conf['rules_loader']) + rules_loader = rules_loader_class(conf) + conf['rules_loader'] = rules_loader + # Make sure we have all the required globals for the loader + # Make sure we have all required globals + if rules_loader.required_globals - frozenset(list(conf.keys())): + raise EAException( + '%s must contain %s' % (filename, ', '.join(rules_loader.required_globals - frozenset(list(conf.keys()))))) - conf['rules'] = rules return conf -def get_rule_hashes(conf, use_rule=None): - rule_files = get_file_paths(conf, use_rule) - rule_mod_times = {} - for rule_file in rule_files: - rule_mod_times[rule_file] = get_rulefile_hash(rule_file) - return rule_mod_times - - -def get_rulefile_hash(rule_file): - rulefile_hash = '' - if os.path.exists(rule_file): - with open(rule_file) as fh: - rulefile_hash = hashlib.sha1(fh.read()).digest() - for import_rule_file in import_rules.get(rule_file, []): - rulefile_hash += get_rulefile_hash(import_rule_file) - return rulefile_hash - - -def adjust_deprecated_values(rule): - # From rename of simple HTTP alerter - if rule.get('type') == 'simple': - rule['type'] = 'post' - if 'simple_proxy' in rule: - rule['http_post_proxy'] = rule['simple_proxy'] - if 'simple_webhook_url' in rule: - rule['http_post_url'] = rule['simple_webhook_url'] - logging.warning('"simple" alerter has been renamed "post" and comptability may be removed in a future release.') +def configure_logging(args, conf): + # configure logging from config file if provided + if 'logging' in conf: + # load new logging config + logging.config.dictConfig(conf['logging']) + + if args.verbose and args.debug: + elastalert_logger.info( + "Note: --debug and --verbose flags are set. --debug takes precedent." + ) + + # re-enable INFO log level on elastalert_logger in verbose/debug mode + # (but don't touch it if it is already set to INFO or below by config) + if args.verbose or args.debug: + if elastalert_logger.level > logging.INFO or elastalert_logger.level == logging.NOTSET: + elastalert_logger.setLevel(logging.INFO) + + if args.debug: + elastalert_logger.info( + """Note: In debug mode, alerts will be logged to console but NOT actually sent. + To send them but remain verbose, use --verbose instead.""" + ) + + if not args.es_debug and 'logging' not in conf: + logging.getLogger('elasticsearch').setLevel(logging.WARNING) + + if args.es_debug_trace: + tracer = logging.getLogger('elasticsearch.trace') + tracer.setLevel(logging.INFO) + tracer.addHandler(logging.FileHandler(args.es_debug_trace)) diff --git a/elastalert/create_index.py b/elastalert/create_index.py index b12ee7e5e..a0858da70 100644 --- a/elastalert/create_index.py +++ b/elastalert/create_index.py @@ -1,25 +1,151 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import print_function - import argparse import getpass +import json import os import time import elasticsearch.helpers import yaml -from auth import Auth from elasticsearch import RequestsHttpConnection from elasticsearch.client import Elasticsearch from elasticsearch.client import IndicesClient from elasticsearch.exceptions import NotFoundError from envparse import Env +from .auth import Auth env = Env(ES_USE_SSL=bool) +def create_index_mappings(es_client, ea_index, recreate=False, old_ea_index=None): + esversion = es_client.info()["version"]["number"] + print("Elastic Version: " + esversion) + + es_index_mappings = read_es_index_mappings() if is_atleastsix(esversion) else read_es_index_mappings(5) + + es_index = IndicesClient(es_client) + if not recreate: + if es_index.exists(ea_index): + print('Index ' + ea_index + ' already exists. Skipping index creation.') + return None + + # (Re-)Create indices. + if is_atleastsix(esversion): + index_names = ( + ea_index, + ea_index + '_status', + ea_index + '_silence', + ea_index + '_error', + ea_index + '_past', + ) + else: + index_names = ( + ea_index, + ) + for index_name in index_names: + if es_index.exists(index_name): + print('Deleting index ' + index_name + '.') + try: + es_index.delete(index_name) + except NotFoundError: + # Why does this ever occur?? It shouldn't. But it does. + pass + es_index.create(index_name) + + # To avoid a race condition. TODO: replace this with a real check + time.sleep(2) + + if is_atleastseven(esversion): + # TODO remove doc_type completely when elasicsearch client allows doc_type=None + # doc_type is a deprecated feature and will be completely removed in Elasicsearch 8 + es_client.indices.put_mapping(index=ea_index, doc_type='_doc', + body=es_index_mappings['elastalert'], include_type_name=True) + es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc', + body=es_index_mappings['elastalert_status'], include_type_name=True) + es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc', + body=es_index_mappings['silence'], include_type_name=True) + es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc', + body=es_index_mappings['elastalert_error'], include_type_name=True) + es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc', + body=es_index_mappings['past_elastalert'], include_type_name=True) + elif is_atleastsixtwo(esversion): + es_client.indices.put_mapping(index=ea_index, doc_type='_doc', + body=es_index_mappings['elastalert']) + es_client.indices.put_mapping(index=ea_index + '_status', doc_type='_doc', + body=es_index_mappings['elastalert_status']) + es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='_doc', + body=es_index_mappings['silence']) + es_client.indices.put_mapping(index=ea_index + '_error', doc_type='_doc', + body=es_index_mappings['elastalert_error']) + es_client.indices.put_mapping(index=ea_index + '_past', doc_type='_doc', + body=es_index_mappings['past_elastalert']) + elif is_atleastsix(esversion): + es_client.indices.put_mapping(index=ea_index, doc_type='elastalert', + body=es_index_mappings['elastalert']) + es_client.indices.put_mapping(index=ea_index + '_status', doc_type='elastalert_status', + body=es_index_mappings['elastalert_status']) + es_client.indices.put_mapping(index=ea_index + '_silence', doc_type='silence', + body=es_index_mappings['silence']) + es_client.indices.put_mapping(index=ea_index + '_error', doc_type='elastalert_error', + body=es_index_mappings['elastalert_error']) + es_client.indices.put_mapping(index=ea_index + '_past', doc_type='past_elastalert', + body=es_index_mappings['past_elastalert']) + else: + es_client.indices.put_mapping(index=ea_index, doc_type='elastalert', + body=es_index_mappings['elastalert']) + es_client.indices.put_mapping(index=ea_index, doc_type='elastalert_status', + body=es_index_mappings['elastalert_status']) + es_client.indices.put_mapping(index=ea_index, doc_type='silence', + body=es_index_mappings['silence']) + es_client.indices.put_mapping(index=ea_index, doc_type='elastalert_error', + body=es_index_mappings['elastalert_error']) + es_client.indices.put_mapping(index=ea_index, doc_type='past_elastalert', + body=es_index_mappings['past_elastalert']) + + print('New index %s created' % ea_index) + if old_ea_index: + print("Copying all data from old index '{0}' to new index '{1}'".format(old_ea_index, ea_index)) + # Use the defaults for chunk_size, scroll, scan_kwargs, and bulk_kwargs + elasticsearch.helpers.reindex(es_client, old_ea_index, ea_index) + + print('Done!') + + +def read_es_index_mappings(es_version=6): + print('Reading Elastic {0} index mappings:'.format(es_version)) + return { + 'silence': read_es_index_mapping('silence', es_version), + 'elastalert_status': read_es_index_mapping('elastalert_status', es_version), + 'elastalert': read_es_index_mapping('elastalert', es_version), + 'past_elastalert': read_es_index_mapping('past_elastalert', es_version), + 'elastalert_error': read_es_index_mapping('elastalert_error', es_version) + } + + +def read_es_index_mapping(mapping, es_version=6): + base_path = os.path.abspath(os.path.dirname(__file__)) + mapping_path = 'es_mappings/{0}/{1}.json'.format(es_version, mapping) + path = os.path.join(base_path, mapping_path) + with open(path, 'r') as f: + print("Reading index mapping '{0}'".format(mapping_path)) + return json.load(f) + + +def is_atleastsix(es_version): + return int(es_version.split(".")[0]) >= 6 + + +def is_atleastsixtwo(es_version): + major, minor = list(map(int, es_version.split(".")[:2])) + return major > 6 or (major == 6 and minor >= 2) + + +def is_atleastseven(es_version): + return int(es_version.split(".")[0]) >= 7 + + def main(): parser = argparse.ArgumentParser() parser.add_argument('--host', default=os.environ.get('ES_HOST', None), help='Elasticsearch host') @@ -31,10 +157,13 @@ def main(): parser.add_argument('--ssl', action='store_true', default=env('ES_USE_SSL', None), help='Use TLS') parser.add_argument('--no-ssl', dest='ssl', action='store_false', help='Do not use TLS') parser.add_argument('--verify-certs', action='store_true', default=None, help='Verify TLS certificates') - parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false', help='Do not verify TLS certificates') + parser.add_argument('--no-verify-certs', dest='verify_certs', action='store_false', + help='Do not verify TLS certificates') parser.add_argument('--index', help='Index name to create') + parser.add_argument('--alias', help='Alias name to create') parser.add_argument('--old-index', help='Old index name to copy') - parser.add_argument('--send_get_body_as', default='GET', help='Method for querying Elasticsearch - POST, GET or source') + parser.add_argument('--send_get_body_as', default='GET', + help='Method for querying Elasticsearch - POST, GET or source') parser.add_argument( '--boto-profile', default=None, @@ -48,21 +177,22 @@ def main(): '--aws-region', default=None, help='AWS Region to use for signing requests. Optionally use the AWS_DEFAULT_REGION environment variable') - parser.add_argument('--timeout', default=60, help='Elasticsearch request timeout') + parser.add_argument('--timeout', default=60, type=int, help='Elasticsearch request timeout') parser.add_argument('--config', default='config.yaml', help='Global config file (default: config.yaml)') - parser.add_argument('--recreate', type=bool, default=False, help='Force re-creation of the index (this will cause data loss).') + parser.add_argument('--recreate', type=bool, default=False, + help='Force re-creation of the index (this will cause data loss).') args = parser.parse_args() - if os.path.isfile('config.yaml'): - filename = 'config.yaml' - elif os.path.isfile(args.config): + if os.path.isfile(args.config): filename = args.config + elif os.path.isfile('../config.yaml'): + filename = '../config.yaml' else: filename = '' if filename: with open(filename) as config_file: - data = yaml.load(config_file) + data = yaml.load(config_file, Loader=yaml.FullLoader) host = args.host if args.host else data.get('es_host') port = args.port if args.port else data.get('es_port') username = args.username if args.username else data.get('es_username') @@ -76,36 +206,41 @@ def main(): client_cert = data.get('client_cert') client_key = data.get('client_key') index = args.index if args.index is not None else data.get('writeback_index') + alias = args.alias if args.alias is not None else data.get('writeback_alias') old_index = args.old_index if args.old_index is not None else None else: username = args.username if args.username else None password = args.password if args.password else None aws_region = args.aws_region - host = args.host if args.host else raw_input('Enter Elasticsearch host: ') - port = args.port if args.port else int(raw_input('Enter Elasticsearch port: ')) + host = args.host if args.host else input('Enter Elasticsearch host: ') + port = args.port if args.port else int(input('Enter Elasticsearch port: ')) use_ssl = (args.ssl if args.ssl is not None - else raw_input('Use SSL? t/f: ').lower() in ('t', 'true')) + else input('Use SSL? t/f: ').lower() in ('t', 'true')) if use_ssl: verify_certs = (args.verify_certs if args.verify_certs is not None - else raw_input('Verify TLS certificates? t/f: ').lower() not in ('f', 'false')) + else input('Verify TLS certificates? t/f: ').lower() not in ('f', 'false')) else: verify_certs = True if args.no_auth is None and username is None: - username = raw_input('Enter optional basic-auth username (or leave blank): ') + username = input('Enter optional basic-auth username (or leave blank): ') password = getpass.getpass('Enter optional basic-auth password (or leave blank): ') url_prefix = (args.url_prefix if args.url_prefix is not None - else raw_input('Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): ')) + else input('Enter optional Elasticsearch URL prefix (prepends a string to the URL of every request): ')) send_get_body_as = args.send_get_body_as ca_certs = None client_cert = None client_key = None - index = args.index if args.index is not None else raw_input('New index name? (Default elastalert_status) ') + index = args.index if args.index is not None else input('New index name? (Default elastalert_status) ') if not index: index = 'elastalert_status' + alias = args.alias if args.alias is not None else input('New alias name? (Default elastalert_alerts) ') + if not alias: + alias = 'elastalert_alias' old_index = (args.old_index if args.old_index is not None - else raw_input('Name of existing index to copy? (Default None) ')) + else input('Name of existing index to copy? (Default None) ')) timeout = args.timeout + auth = Auth() http_auth = auth(host=host, username=username, @@ -126,151 +261,7 @@ def main(): ca_certs=ca_certs, client_key=client_key) - esversion = es.info()["version"]["number"] - print("Elastic Version:" + esversion.split(".")[0]) - elasticversion = int(esversion.split(".")[0]) - - if(elasticversion > 5): - mapping = {'type': 'keyword'} - else: - mapping = {'index': 'not_analyzed', 'type': 'string'} - - print("Mapping used for string:" + str(mapping)) - - silence_mapping = { - 'silence': { - 'properties': { - 'rule_name': mapping, - 'until': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - }, - }, - } - ess_mapping = { - 'elastalert_status': { - 'properties': { - 'rule_name': mapping, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - }, - }, - } - es_mapping = { - 'elastalert': { - 'properties': { - 'rule_name': mapping, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - 'alert_time': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - 'match_time': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - 'match_body': { - 'type': 'object', - 'enabled': False, - }, - 'aggregate_id': mapping, - }, - }, - } - past_mapping = { - 'past_elastalert': { - 'properties': { - 'rule_name': mapping, - 'match_body': { - 'type': 'object', - 'enabled': False, - }, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - 'aggregate_id': mapping, - }, - }, - } - error_mapping = { - 'elastalert_error': { - 'properties': { - 'data': { - 'type': 'object', - 'enabled': False, - }, - '@timestamp': { - 'type': 'date', - 'format': 'dateOptionalTime', - }, - }, - }, - } - - es_index = IndicesClient(es) - if not args.recreate: - if es_index.exists(index): - print('Index ' + index + ' already exists. Skipping index creation.') - return None - - # (Re-)Create indices. - if (elasticversion > 5): - index_names = ( - index, - index + '_status', - index + '_silence', - index + '_error', - index + '_past', - ) - else: - index_names = ( - index, - ) - for index_name in index_names: - if es_index.exists(index_name): - print('Deleting index ' + index_name + '.') - try: - es_index.delete(index_name) - except NotFoundError: - # Why does this ever occur?? It shouldn't. But it does. - pass - es_index.create(index_name) - - # To avoid a race condition. TODO: replace this with a real check - time.sleep(2) - - if(elasticversion > 5): - es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) - es.indices.put_mapping(index=index + '_status', doc_type='elastalert_status', body=ess_mapping) - es.indices.put_mapping(index=index + '_silence', doc_type='silence', body=silence_mapping) - es.indices.put_mapping(index=index + '_error', doc_type='elastalert_error', body=error_mapping) - es.indices.put_mapping(index=index + '_past', doc_type='past_elastalert', body=past_mapping) - print('New index %s created' % index) - else: - es.indices.put_mapping(index=index, doc_type='elastalert', body=es_mapping) - es.indices.put_mapping(index=index, doc_type='elastalert_status', body=ess_mapping) - es.indices.put_mapping(index=index, doc_type='silence', body=silence_mapping) - es.indices.put_mapping(index=index, doc_type='elastalert_error', body=error_mapping) - es.indices.put_mapping(index=index, doc_type='past_elastalert', body=past_mapping) - print('New index %s created' % index) - - if old_index: - print("Copying all data from old index '{0}' to new index '{1}'".format(old_index, index)) - # Use the defaults for chunk_size, scroll, scan_kwargs, and bulk_kwargs - elasticsearch.helpers.reindex(es, old_index, index) - - print('Done!') + create_index_mappings(es_client=es, ea_index=index, recreate=args.recreate, old_ea_index=old_index) if __name__ == '__main__': diff --git a/elastalert/elastalert.py b/elastalert/elastalert.py index 721523776..b078c86db 100755 --- a/elastalert/elastalert.py +++ b/elastalert/elastalert.py @@ -5,8 +5,10 @@ import json import logging import os +import random import signal import sys +import threading import time import timeit import traceback @@ -16,41 +18,44 @@ from socket import error import dateutil.tz -import kibana -import yaml -from alerts import DebugAlerter -from config import get_rule_hashes -from config import load_configuration -from config import load_rules +import pytz +from apscheduler.schedulers.background import BackgroundScheduler from croniter import croniter from elasticsearch.exceptions import ConnectionError from elasticsearch.exceptions import ElasticsearchException +from elasticsearch.exceptions import NotFoundError from elasticsearch.exceptions import TransportError -from enhancements import DropMatchException -from ruletypes import FlatlineRule -from util import add_raw_postfix -from util import cronite_datetime_to_timestamp -from util import dt_to_ts -from util import dt_to_unix -from util import EAException -from util import elastalert_logger -from util import elasticsearch_client -from util import format_index -from util import lookup_es_key -from util import parse_deadline -from util import parse_duration -from util import pretty_ts -from util import replace_dots_in_field_names -from util import seconds -from util import set_es_key -from util import total_seconds -from util import ts_add -from util import ts_now -from util import ts_to_dt -from util import unix_to_dt - - -class ElastAlerter(): + +from . import kibana +from .alerts import DebugAlerter +from .config import load_conf +from .enhancements import DropMatchException +from .kibana_discover import generate_kibana_discover_url +from .ruletypes import FlatlineRule +from .util import add_raw_postfix +from .util import cronite_datetime_to_timestamp +from .util import dt_to_ts +from .util import dt_to_unix +from .util import EAException +from .util import elastalert_logger +from .util import elasticsearch_client +from .util import format_index +from .util import lookup_es_key +from .util import parse_deadline +from .util import parse_duration +from .util import pretty_ts +from .util import replace_dots_in_field_names +from .util import seconds +from .util import set_es_key +from .util import should_scrolling_continue +from .util import total_seconds +from .util import ts_add +from .util import ts_now +from .util import ts_to_dt +from .util import unix_to_dt + + +class ElastAlerter(object): """ The main ElastAlert runner. This class holds all state about active rules, controls when queries are run, and passes information between rules and alerts. @@ -63,6 +68,8 @@ class ElastAlerter(): should not be passed directly from a configuration file, but must be populated by config.py:load_rules instead. """ + thread_data = threading.local() + def parse_args(self, args): parser = argparse.ArgumentParser() parser.add_argument( @@ -101,6 +108,7 @@ def parse_args(self, args): self.args = parser.parse_args(args) def __init__(self, args): + self.es_clients = {} self.parse_args(args) self.debug = self.args.debug self.verbose = self.args.verbose @@ -127,11 +135,16 @@ def __init__(self, args): tracer.setLevel(logging.INFO) tracer.addHandler(logging.FileHandler(self.args.es_debug_trace)) - self.conf = load_rules(self.args) + self.conf = load_conf(self.args) + self.rules_loader = self.conf['rules_loader'] + self.rules = self.rules_loader.load(self.conf, self.args) + + print(len(self.rules), 'rules loaded') + self.max_query_size = self.conf['max_query_size'] self.scroll_keepalive = self.conf['scroll_keepalive'] - self.rules = self.conf['rules'] self.writeback_index = self.conf['writeback_index'] + self.writeback_alias = self.conf['writeback_alias'] self.run_every = self.conf['run_every'] self.alert_time_limit = self.conf['alert_time_limit'] self.old_query_limit = self.conf['old_query_limit'] @@ -140,47 +153,30 @@ def __init__(self, args): self.from_addr = self.conf.get('from_addr', 'ElastAlert') self.smtp_host = self.conf.get('smtp_host', 'localhost') self.max_aggregation = self.conf.get('max_aggregation', 10000) - self.alerts_sent = 0 - self.num_hits = 0 - self.num_dupes = 0 - self.current_es = None - self.current_es_addr = None self.buffer_time = self.conf['buffer_time'] self.silence_cache = {} - self.rule_hashes = get_rule_hashes(self.conf, self.args.rule) + self.rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) self.starttime = self.args.start self.disabled_rules = [] self.replace_dots_in_field_names = self.conf.get('replace_dots_in_field_names', False) + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.scheduler = BackgroundScheduler() self.string_multi_field_name = self.conf.get('string_multi_field_name', False) + self.add_metadata_alert = self.conf.get('add_metadata_alert', False) + self.show_disabled_rules = self.conf.get('show_disabled_rules', True) self.writeback_es = elasticsearch_client(self.conf) - self._es_version = None remove = [] for rule in self.rules: if not self.init_rule(rule): remove.append(rule) - map(self.rules.remove, remove) + list(map(self.rules.remove, remove)) if self.args.silence: self.silence() - def get_version(self): - info = self.writeback_es.info() - return info['version']['number'] - - @property - def es_version(self): - if self._es_version is None: - self._es_version = self.get_version() - return self._es_version - - def is_atleastfive(self): - return int(self.es_version.split(".")[0]) >= 5 - - def is_atleastsix(self): - return int(self.es_version.split(".")[0]) >= 6 - @staticmethod def get_index(rule, starttime=None, endtime=None): """ Gets the index for a rule. If strftime is set and starttime and endtime @@ -200,21 +196,6 @@ def get_index(rule, starttime=None, endtime=None): else: return index - def get_six_index(self, doc_type): - """ In ES6, you cannot have multiple _types per index, - therefore we use self.writeback_index as the prefix for the actual - index name, based on doc_type. """ - writeback_index = self.writeback_index - if doc_type == 'silence': - writeback_index += '_silence' - elif doc_type == 'past_elastalert': - writeback_index += '_past' - elif doc_type == 'elastalert_status': - writeback_index += '_status' - elif doc_type == 'elastalert_error': - writeback_index += '_error' - return writeback_index - @staticmethod def get_query(filters, starttime=None, endtime=None, sort=True, timestamp_field='@timestamp', to_ts_func=dt_to_ts, desc=False, five=False): @@ -242,17 +223,21 @@ def get_query(filters, starttime=None, endtime=None, sort=True, timestamp_field= query['sort'] = [{timestamp_field: {'order': 'desc' if desc else 'asc'}}] return query - def get_terms_query(self, query, size, field, five=False): + def get_terms_query(self, query, rule, size, field, five=False): """ Takes a query generated by get_query and outputs a aggregation query """ query_element = query['query'] if 'sort' in query_element: query_element.pop('sort') if not five: - query_element['filtered'].update({'aggs': {'counts': {'terms': {'field': field, 'size': size}}}}) + query_element['filtered'].update({'aggs': {'counts': {'terms': {'field': field, + 'size': size, + 'min_doc_count': rule.get('min_doc_count', 1)}}}}) aggs_query = {'aggs': query_element} else: aggs_query = query - aggs_query['aggs'] = {'counts': {'terms': {'field': field, 'size': size}}} + aggs_query['aggs'] = {'counts': {'terms': {'field': field, + 'size': size, + 'min_doc_count': rule.get('min_doc_count', 1)}}} return aggs_query def get_aggregation_query(self, query, rule, query_key, terms_size, timestamp_field='@timestamp'): @@ -279,7 +264,9 @@ def get_aggregation_query(self, query, rule, query_key, terms_size, timestamp_fi if query_key is not None: for idx, key in reversed(list(enumerate(query_key.split(',')))): - aggs_element = {'bucket_aggs': {'terms': {'field': key, 'size': terms_size}, 'aggs': aggs_element}} + aggs_element = {'bucket_aggs': {'terms': {'field': key, 'size': terms_size, + 'min_doc_count': rule.get('min_doc_count', 1)}, + 'aggs': aggs_element}} if not rule['five']: query_element['filtered'].update({'aggs': aggs_element}) @@ -297,7 +284,12 @@ def get_index_start(self, index, timestamp_field='@timestamp'): """ query = {'sort': {timestamp_field: {'order': 'asc'}}} try: - res = self.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field], ignore_unavailable=True) + if self.thread_data.current_es.is_atleastsixsix(): + res = self.thread_data.current_es.search(index=index, size=1, body=query, + _source_includes=[timestamp_field], ignore_unavailable=True) + else: + res = self.thread_data.current_es.search(index=index, size=1, body=query, _source_include=[timestamp_field], + ignore_unavailable=True) except ElasticsearchException as e: self.handle_error("Elasticsearch query error: %s" % (e), {'index': index, 'query': query}) return '1969-12-30T00:00:00Z' @@ -320,7 +312,7 @@ def process_hits(rule, hits): for hit in hits: # Merge fields and _source hit.setdefault('_source', {}) - for key, value in hit.get('fields', {}).items(): + for key, value in list(hit.get('fields', {}).items()): # Fields are returned as lists, assume any with length 1 are not arrays in _source # Except sometimes they aren't lists. This is dependent on ES version hit['_source'].setdefault(key, value[0] if type(value) is list and len(value) == 1 else value) @@ -342,11 +334,11 @@ def process_hits(rule, hits): if rule.get('compound_query_key'): values = [lookup_es_key(hit['_source'], key) for key in rule['compound_query_key']] - hit['_source'][rule['query_key']] = ', '.join([unicode(value) for value in values]) + hit['_source'][rule['query_key']] = ', '.join([str(value) for value in values]) if rule.get('compound_aggregation_key'): values = [lookup_es_key(hit['_source'], key) for key in rule['compound_aggregation_key']] - hit['_source'][rule['aggregation_key']] = ', '.join([unicode(value) for value in values]) + hit['_source'][rule['aggregation_key']] = ', '.join([str(value) for value in values]) processed_hits.append(hit['_source']) @@ -368,7 +360,10 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): to_ts_func=rule['dt_to_ts'], five=rule['five'], ) - extra_args = {'_source_include': rule['include']} + if self.thread_data.current_es.is_atleastsixsix(): + extra_args = {'_source_includes': rule['include']} + else: + extra_args = {'_source_include': rule['include']} scroll_keepalive = rule.get('scroll_keepalive', self.scroll_keepalive) if not rule.get('_source_enabled'): if rule['five']: @@ -379,9 +374,9 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): try: if scroll: - res = self.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive) + res = self.thread_data.current_es.scroll(scroll_id=rule['scroll_id'], scroll=scroll_keepalive) else: - res = self.current_es.search( + res = self.thread_data.current_es.search( scroll=scroll_keepalive, index=index, size=rule.get('max_query_size', self.max_query_size), @@ -389,7 +384,23 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): ignore_unavailable=True, **extra_args ) - self.total_hits = int(res['hits']['total']) + if '_scroll_id' in res: + rule['scroll_id'] = res['_scroll_id'] + + if self.thread_data.current_es.is_atleastseven(): + self.thread_data.total_hits = int(res['hits']['total']['value']) + else: + self.thread_data.total_hits = int(res['hits']['total']) + + if len(res.get('_shards', {}).get('failures', [])) > 0: + try: + errs = [e['reason']['reason'] for e in res['_shards']['failures'] if 'Failed to parse' in e['reason']['reason']] + if len(errs): + raise ElasticsearchException(errs) + except (TypeError, KeyError): + # Different versions of ES have this formatted in different ways. Fallback to str-ing the whole thing + raise ElasticsearchException(str(res['_shards']['failures'])) + logging.debug(str(res)) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages @@ -399,18 +410,17 @@ def get_hits(self, rule, starttime, endtime, index, scroll=False): self.handle_error('Error running query: %s' % (e), {'rule': rule['name'], 'query': query}) return None hits = res['hits']['hits'] - self.num_hits += len(hits) + self.thread_data.num_hits += len(hits) lt = rule.get('use_local_time') status_log = "Queried rule %s from %s to %s: %s / %s hits" % ( rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), - self.num_hits, + self.thread_data.num_hits, len(hits) ) - if self.total_hits > rule.get('max_query_size', self.max_query_size): + if self.thread_data.total_hits > rule.get('max_query_size', self.max_query_size): elastalert_logger.info("%s (scrolling..)" % status_log) - rule['scroll_id'] = res['_scroll_id'] else: elastalert_logger.info(status_log) @@ -442,7 +452,7 @@ def get_hits_count(self, rule, starttime, endtime, index): ) try: - res = self.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True) + res = self.thread_data.current_es.count(index=index, doc_type=rule['doc_type'], body=query, ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -451,7 +461,7 @@ def get_hits_count(self, rule, starttime, endtime, index): self.handle_error('Error running count query: %s' % (e), {'rule': rule['name'], 'query': query}) return None - self.num_hits += res['count'] + self.thread_data.num_hits += res['count'] lt = rule.get('use_local_time') elastalert_logger.info( "Queried rule %s from %s to %s: %s hits" % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), res['count']) @@ -461,7 +471,7 @@ def get_hits_count(self, rule, starttime, endtime, index): def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=None): rule_filter = copy.copy(rule['filter']) if qk: - qk_list = qk.split(", ") + qk_list = qk.split(",") end = None if rule['five']: end = '.keyword' @@ -493,11 +503,11 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non ) if size is None: size = rule.get('terms_size', 50) - query = self.get_terms_query(base_query, size, key, rule['five']) + query = self.get_terms_query(base_query, rule, size, key, rule['five']) try: if not rule['five']: - res = self.current_es.search( + res = self.thread_data.current_es.deprecated_search( index=index, doc_type=rule['doc_type'], body=query, @@ -505,7 +515,8 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule['doc_type'], body=query, size=0, ignore_unavailable=True) + res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule['doc_type'], + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: # Elasticsearch sometimes gives us GIGANTIC error messages # (so big that they will fill the entire terminal buffer) @@ -520,7 +531,7 @@ def get_hits_terms(self, rule, starttime, endtime, index, key, qk=None, size=Non buckets = res['aggregations']['filtered']['counts']['buckets'] else: buckets = res['aggregations']['counts']['buckets'] - self.num_hits += len(buckets) + self.thread_data.num_hits += len(buckets) lt = rule.get('use_local_time') elastalert_logger.info( 'Queried rule %s from %s to %s: %s buckets' % (rule['name'], pretty_ts(starttime, lt), pretty_ts(endtime, lt), len(buckets)) @@ -543,7 +554,7 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ query = self.get_aggregation_query(base_query, rule, query_key, term_size, rule['timestamp_field']) try: if not rule['five']: - res = self.current_es.search( + res = self.thread_data.current_es.deprecated_search( index=index, doc_type=rule.get('doc_type'), body=query, @@ -551,7 +562,8 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ ignore_unavailable=True ) else: - res = self.current_es.search(index=index, doc_type=rule.get('doc_type'), body=query, size=0, ignore_unavailable=True) + res = self.thread_data.current_es.deprecated_search(index=index, doc_type=rule.get('doc_type'), + body=query, size=0, ignore_unavailable=True) except ElasticsearchException as e: if len(str(e)) > 1024: e = str(e)[:1024] + '... (%d characters removed)' % (len(str(e)) - 1024) @@ -563,7 +575,12 @@ def get_hits_aggregation(self, rule, starttime, endtime, index, query_key, term_ payload = res['aggregations']['filtered'] else: payload = res['aggregations'] - self.num_hits += res['hits']['total'] + + if self.thread_data.current_es.is_atleastseven(): + self.thread_data.num_hits += res['hits']['total']['value'] + else: + self.thread_data.num_hits += res['hits']['total'] + return {endtime: payload} def remove_duplicate_events(self, data, rule): @@ -585,10 +602,10 @@ def remove_old_events(self, rule): buffer_time = rule.get('buffer_time', self.buffer_time) if rule.get('query_delay'): buffer_time += rule['query_delay'] - for _id, timestamp in rule['processed_hits'].iteritems(): + for _id, timestamp in rule['processed_hits'].items(): if now - timestamp > buffer_time: remove.append(_id) - map(rule['processed_hits'].pop, remove) + list(map(rule['processed_hits'].pop, remove)) def run_query(self, rule, start=None, end=None, scroll=False): """ Query for the rule and pass all of the results to the RuleType instance. @@ -605,6 +622,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): # Reset hit counter and query rule_inst = rule['type'] + rule['scrolling_cycle'] = rule.get('scrolling_cycle', 0) + 1 index = self.get_index(rule, start, end) if rule.get('use_count_query'): data = self.get_hits_count(rule, start, end, index) @@ -617,7 +635,7 @@ def run_query(self, rule, start=None, end=None, scroll=False): if data: old_len = len(data) data = self.remove_duplicate_events(data, rule) - self.num_dupes += old_len - len(data) + self.thread_data.num_dupes += old_len - len(data) # There was an exception while querying if data is None: @@ -633,14 +651,19 @@ def run_query(self, rule, start=None, end=None, scroll=False): rule_inst.add_data(data) try: - if rule.get('scroll_id') and self.num_hits < self.total_hits: - self.run_query(rule, start, end, scroll=True) + if rule.get('scroll_id') and self.thread_data.num_hits < self.thread_data.total_hits and should_scrolling_continue(rule): + if not self.run_query(rule, start, end, scroll=True): + return False except RuntimeError: # It's possible to scroll far enough to hit max recursive depth pass if 'scroll_id' in rule: - rule.pop('scroll_id') + scroll_id = rule.pop('scroll_id') + try: + self.thread_data.current_es.clear_scroll(scroll_id=scroll_id) + except NotFoundError: + pass return True @@ -652,18 +675,23 @@ def get_starttime(self, rule): """ sort = {'sort': {'@timestamp': {'order': 'desc'}}} query = {'filter': {'term': {'rule_name': '%s' % (rule['name'])}}} - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): query = {'query': {'bool': query}} query.update(sort) try: - if self.is_atleastsix(): - index = self.get_six_index('elastalert_status') - res = self.writeback_es.search(index=index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) + doc_type = 'elastalert_status' + index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type) + if self.writeback_es.is_atleastsixtwo(): + if self.writeback_es.is_atleastsixsix(): + res = self.writeback_es.search(index=index, size=1, body=query, + _source_includes=['endtime', 'rule_name']) + else: + res = self.writeback_es.search(index=index, size=1, body=query, + _source_include=['endtime', 'rule_name']) else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='elastalert_status', - size=1, body=query, _source_include=['endtime', 'rule_name']) + res = self.writeback_es.deprecated_search(index=index, doc_type=doc_type, + size=1, body=query, _source_include=['endtime', 'rule_name']) if res['hits']['hits']: endtime = ts_to_dt(res['hits']['hits'][0]['_source']['endtime']) @@ -677,7 +705,6 @@ def get_starttime(self, rule): def set_starttime(self, rule, endtime): """ Given a rule and an endtime, sets the appropriate starttime for it. """ - # This means we are starting fresh if 'starttime' not in rule: if not rule.get('scan_entire_timeframe'): @@ -703,9 +730,8 @@ def set_starttime(self, rule, endtime): if 'minimum_starttime' in rule and rule['minimum_starttime'] > buffer_delta: rule['starttime'] = rule['minimum_starttime'] # If buffer_time doesn't bring us past the previous endtime, use that instead - elif 'previous_endtime' in rule: - if rule['previous_endtime'] < buffer_delta: - rule['starttime'] = rule['previous_endtime'] + elif 'previous_endtime' in rule and rule['previous_endtime'] < buffer_delta: + rule['starttime'] = rule['previous_endtime'] self.adjust_start_time_for_overlapping_agg_query(rule) else: rule['starttime'] = buffer_delta @@ -759,7 +785,7 @@ def get_query_key_value(self, rule, match): # get the value for the match's query_key (or none) to form the key used for the silence_cache. # Flatline ruletype sets "key" instead of the actual query_key if isinstance(rule['type'], FlatlineRule) and 'key' in match: - return unicode(match['key']) + return str(match['key']) return self.get_named_key_value(rule, match, 'query_key') def get_aggregation_key_value(self, rule, match): @@ -774,7 +800,7 @@ def get_named_key_value(self, rule, match, key_name): if key_value is not None: # Only do the unicode conversion if we actually found something) # otherwise we might transform None --> 'None' - key_value = unicode(key_value) + key_value = str(key_value) except KeyError: # Some matches may not have the specified key # use a special token for these @@ -802,13 +828,19 @@ def enhance_filter(self, rule): return filters = rule['filter'] - additional_terms = [(rule['compare_key'] + ':"' + term + '"') for term in rule[listname]] + additional_terms = [] + for term in rule[listname]: + if not term.startswith('/') or not term.endswith('/'): + additional_terms.append(rule['compare_key'] + ':"' + term + '"') + else: + # These are regular expressions and won't work if they are quoted + additional_terms.append(rule['compare_key'] + ':' + term) if listname == 'whitelist': query = "NOT " + " AND NOT ".join(additional_terms) else: query = " OR ".join(additional_terms) query_str_filter = {'query_string': {'query': query}} - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): filters.append(query_str_filter) else: filters.append({'query': query_str_filter}) @@ -823,9 +855,7 @@ def run_rule(self, rule, endtime, starttime=None): :return: The number of matches that the rule produced. """ run_start = time.time() - - self.current_es = elasticsearch_client(rule) - self.current_es_addr = (rule['es_host'], rule['es_port']) + self.thread_data.current_es = self.es_clients.setdefault(rule['name'], elasticsearch_client(rule)) # If there are pending aggregate matches, try processing them for x in range(len(rule['agg_matches'])): @@ -839,6 +869,7 @@ def run_rule(self, rule, endtime, starttime=None): self.set_starttime(rule, endtime) rule['original_starttime'] = rule['starttime'] + rule['scrolling_cycle'] = 0 # Don't run if starttime was set to the future if ts_now() <= rule['starttime']: @@ -846,9 +877,9 @@ def run_rule(self, rule, endtime, starttime=None): return 0 # Run the rule. If querying over a large time period, split it up into segments - self.num_hits = 0 - self.num_dupes = 0 - self.cumulative_hits = 0 + self.thread_data.num_hits = 0 + self.thread_data.num_dupes = 0 + self.thread_data.cumulative_hits = 0 segment_size = self.get_segment_size(rule) tmp_endtime = rule['starttime'] @@ -857,14 +888,16 @@ def run_rule(self, rule, endtime, starttime=None): tmp_endtime = tmp_endtime + segment_size if not self.run_query(rule, rule['starttime'], tmp_endtime): return 0 - self.cumulative_hits += self.num_hits - self.num_hits = 0 + self.thread_data.cumulative_hits += self.thread_data.num_hits + self.thread_data.num_hits = 0 rule['starttime'] = tmp_endtime rule['type'].garbage_collect(tmp_endtime) if rule.get('aggregation_query_element'): if endtime - tmp_endtime == segment_size: - self.run_query(rule, tmp_endtime, endtime) + if not self.run_query(rule, tmp_endtime, endtime): + return 0 + self.thread_data.cumulative_hits += self.thread_data.num_hits elif total_seconds(rule['original_starttime'] - tmp_endtime) == 0: rule['starttime'] = rule['original_starttime'] return 0 @@ -873,13 +906,14 @@ def run_rule(self, rule, endtime, starttime=None): else: if not self.run_query(rule, rule['starttime'], endtime): return 0 + self.thread_data.cumulative_hits += self.thread_data.num_hits rule['type'].garbage_collect(endtime) # Process any new matches num_matches = len(rule['type'].matches) while rule['type'].matches: match = rule['type'].matches.pop(0) - match['num_hits'] = self.cumulative_hits + match['num_hits'] = self.thread_data.cumulative_hits match['num_matches'] = num_matches # If realert is set, silence the rule for that duration @@ -925,7 +959,7 @@ def run_rule(self, rule, endtime, starttime=None): 'endtime': endtime, 'starttime': rule['original_starttime'], 'matches': num_matches, - 'hits': max(self.num_hits, self.cumulative_hits), + 'hits': max(self.thread_data.num_hits, self.thread_data.cumulative_hits), '@timestamp': ts_now(), 'time_taken': time_taken} self.writeback('elastalert_status', body) @@ -934,6 +968,9 @@ def run_rule(self, rule, endtime, starttime=None): def init_rule(self, new_rule, new=True): ''' Copies some necessary non-config state from an exiting rule to a new rule. ''' + if not new: + self.scheduler.remove_job(job_id=new_rule['name']) + try: self.modify_rule_for_ES5(new_rule) except TransportError as e: @@ -948,7 +985,7 @@ def init_rule(self, new_rule, new=True): if 'top_count_keys' in new_rule and new_rule.get('raw_count_keys', True): if self.string_multi_field_name: string_multi_field_name = self.string_multi_field_name - elif self.is_atleastfive(): + elif self.writeback_es.is_atleastfive(): string_multi_field_name = '.keyword' else: string_multi_field_name = '.raw' @@ -968,7 +1005,9 @@ def init_rule(self, new_rule, new=True): blank_rule = {'agg_matches': [], 'aggregate_alert_time': {}, 'current_aggregate_id': {}, - 'processed_hits': {}} + 'processed_hits': {}, + 'run_every': self.run_every, + 'has_run_once': False} rule = blank_rule # Set rule to either a blank template or existing rule with same name @@ -984,19 +1023,28 @@ def init_rule(self, new_rule, new=True): 'aggregate_alert_time', 'processed_hits', 'starttime', - 'minimum_starttime'] + 'minimum_starttime', + 'has_run_once'] for prop in copy_properties: if prop not in rule: continue new_rule[prop] = rule[prop] + job = self.scheduler.add_job(self.handle_rule_execution, 'interval', + args=[new_rule], + seconds=new_rule['run_every'].total_seconds(), + id=new_rule['name'], + max_instances=1, + jitter=5) + job.modify(next_run_time=datetime.datetime.now() + datetime.timedelta(seconds=random.randint(0, 15))) + return new_rule @staticmethod def modify_rule_for_ES5(new_rule): # Get ES version per rule rule_es = elasticsearch_client(new_rule) - if int(rule_es.info()['version']['number'].split(".")[0]) >= 5: + if rule_es.is_atleastfive(): new_rule['five'] = True else: new_rule['five'] = False @@ -1012,21 +1060,30 @@ def modify_rule_for_ES5(new_rule): new_rule['filter'] = new_filters def load_rule_changes(self): - ''' Using the modification times of rule config files, syncs the running rules - to match the files in rules_folder by removing, adding or reloading rules. ''' - new_rule_hashes = get_rule_hashes(self.conf, self.args.rule) + """ Using the modification times of rule config files, syncs the running rules + to match the files in rules_folder by removing, adding or reloading rules. """ + new_rule_hashes = self.rules_loader.get_hashes(self.conf, self.args.rule) # Check each current rule for changes - for rule_file, hash_value in self.rule_hashes.iteritems(): + for rule_file, hash_value in self.rule_hashes.items(): if rule_file not in new_rule_hashes: # Rule file was deleted elastalert_logger.info('Rule file %s not found, stopping rule execution' % (rule_file)) - self.rules = [rule for rule in self.rules if rule['rule_file'] != rule_file] + for rule in self.rules: + if rule['rule_file'] == rule_file: + break + else: + continue + self.scheduler.remove_job(job_id=rule['name']) + self.rules.remove(rule) continue if hash_value != new_rule_hashes[rule_file]: # Rule file was changed, reload rule try: - new_rule = load_configuration(rule_file, self.conf) + new_rule = self.rules_loader.load_configuration(rule_file, self.conf) + if not new_rule: + logging.error('Invalid rule file skipped: %s' % rule_file) + continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: elastalert_logger.info('Rule file %s is now disabled.' % (rule_file)) # Remove this rule if it's been disabled @@ -1036,12 +1093,11 @@ def load_rule_changes(self): message = 'Could not load rule %s: %s' % (rule_file, e) self.handle_error(message) # Want to send email to address specified in the rule. Try and load the YAML to find it. - with open(rule_file) as f: - try: - rule_yaml = yaml.load(f) - except yaml.scanner.ScannerError: - self.send_notification_email(exception=e) - continue + try: + rule_yaml = self.rules_loader.load_yaml(rule_file) + except EAException: + self.send_notification_email(exception=e) + continue self.send_notification_email(exception=e, rule=rule_yaml) continue @@ -1064,7 +1120,10 @@ def load_rule_changes(self): if not self.args.rule: for rule_file in set(new_rule_hashes.keys()) - set(self.rule_hashes.keys()): try: - new_rule = load_configuration(rule_file, self.conf) + new_rule = self.rules_loader.load_configuration(rule_file, self.conf) + if not new_rule: + logging.error('Invalid rule file skipped: %s' % rule_file) + continue if 'is_enabled' in new_rule and not new_rule['is_enabled']: continue if new_rule['name'] in [rule['name'] for rule in self.rules]: @@ -1075,6 +1134,8 @@ def load_rule_changes(self): continue if self.init_rule(new_rule): elastalert_logger.info('Loaded new rule %s' % (rule_file)) + if new_rule['name'] in self.es_clients: + self.es_clients.pop(new_rule['name']) self.rules.append(new_rule) self.rule_hashes = new_rule_hashes @@ -1090,14 +1151,20 @@ def start(self): except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (self.starttime)) exit(1) + + for rule in self.rules: + rule['initial_starttime'] = self.starttime self.wait_until_responsive(timeout=self.args.timeout) self.running = True elastalert_logger.info("Starting up") + self.scheduler.add_job(self.handle_pending_alerts, 'interval', + seconds=self.run_every.total_seconds(), id='_internal_handle_pending_alerts') + self.scheduler.add_job(self.handle_config_change, 'interval', + seconds=self.run_every.total_seconds(), id='_internal_handle_config_change') + self.scheduler.start() while self.running: next_run = datetime.datetime.utcnow() + self.run_every - self.run_all_rules() - # Quit after end_time has been reached if self.args.end: endtime = ts_to_dt(self.args.end) @@ -1108,6 +1175,10 @@ def start(self): if next_run < datetime.datetime.utcnow(): continue + # Show disabled rules + if self.show_disabled_rules: + elastalert_logger.info("Disabled rules are: %s" % (str(self.get_disabled_rules()))) + # Wait before querying again sleep_duration = total_seconds(next_run - datetime.datetime.utcnow()) self.sleep_for(sleep_duration) @@ -1127,7 +1198,7 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): ref = clock() while (clock() - ref) < timeout: try: - if self.writeback_es.indices.exists(self.writeback_index): + if self.writeback_es.indices.exists(self.writeback_alias): return except ConnectionError: pass @@ -1135,8 +1206,8 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): if self.writeback_es.ping(): logging.error( - 'Writeback index "%s" does not exist, did you run `elastalert-create-index`?', - self.writeback_index, + 'Writeback alias "%s" does not exist, did you run `elastalert-create-index`?', + self.writeback_alias, ) else: logging.error( @@ -1148,58 +1219,104 @@ def wait_until_responsive(self, timeout, clock=timeit.default_timer): def run_all_rules(self): """ Run each rule one time """ + self.handle_pending_alerts() + + for rule in self.rules: + self.handle_rule_execution(rule) + + self.handle_config_change() + + def handle_pending_alerts(self): + self.thread_data.alerts_sent = 0 self.send_pending_alerts() + elastalert_logger.info("Background alerts thread %s pending alerts sent at %s" % (self.thread_data.alerts_sent, + pretty_ts(ts_now()))) - next_run = datetime.datetime.utcnow() + self.run_every + def handle_config_change(self): + if not self.args.pin_rules: + self.load_rule_changes() + elastalert_logger.info("Background configuration change check run at %s" % (pretty_ts(ts_now()))) + + def handle_rule_execution(self, rule): + self.thread_data.alerts_sent = 0 + next_run = datetime.datetime.utcnow() + rule['run_every'] + # Set endtime based on the rule's delay + delay = rule.get('query_delay') + if hasattr(self.args, 'end') and self.args.end: + endtime = ts_to_dt(self.args.end) + elif delay: + endtime = ts_now() - delay + else: + endtime = ts_now() + + # Apply rules based on execution time limits + if rule.get('limit_execution'): + rule['next_starttime'] = None + rule['next_min_starttime'] = None + exec_next = next(croniter(rule['limit_execution'])) + endtime_epoch = dt_to_unix(endtime) + # If the estimated next endtime (end + run_every) isn't at least a minute past the next exec time + # That means that we need to pause execution after this run + if endtime_epoch + rule['run_every'].total_seconds() < exec_next - 59: + # apscheduler requires pytz tzinfos, so don't use unix_to_dt here! + rule['next_starttime'] = datetime.datetime.utcfromtimestamp(exec_next).replace(tzinfo=pytz.utc) + if rule.get('limit_execution_coverage'): + rule['next_min_starttime'] = rule['next_starttime'] + if not rule['has_run_once']: + self.reset_rule_schedule(rule) + return - for rule in self.rules: - # Set endtime based on the rule's delay - delay = rule.get('query_delay') - if hasattr(self.args, 'end') and self.args.end: - endtime = ts_to_dt(self.args.end) - elif delay: - endtime = ts_now() - delay - else: - endtime = ts_now() + rule['has_run_once'] = True + try: + num_matches = self.run_rule(rule, endtime, rule.get('initial_starttime')) + except EAException as e: + self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']}) + except Exception as e: + self.handle_uncaught_exception(e, rule) + else: + old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time')) + elastalert_logger.info("Ran %s from %s to %s: %s query hits (%s already seen), %s matches," + " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')), + self.thread_data.num_hits, self.thread_data.num_dupes, num_matches, + self.thread_data.alerts_sent)) + self.thread_data.alerts_sent = 0 - try: - num_matches = self.run_rule(rule, endtime, self.starttime) - except EAException as e: - self.handle_error("Error running rule %s: %s" % (rule['name'], e), {'rule': rule['name']}) - except Exception as e: - self.handle_uncaught_exception(e, rule) - else: - old_starttime = pretty_ts(rule.get('original_starttime'), rule.get('use_local_time')) - total_hits = max(self.num_hits, self.cumulative_hits) - elastalert_logger.info("Ran %s from %s to %s: %s query hits (%s already seen), %s matches," - " %s alerts sent" % (rule['name'], old_starttime, pretty_ts(endtime, rule.get('use_local_time')), - total_hits, self.num_dupes, num_matches, self.alerts_sent)) - self.alerts_sent = 0 - - if next_run < datetime.datetime.utcnow(): - # We were processing for longer than our refresh interval - # This can happen if --start was specified with a large time period - # or if we are running too slow to process events in real time. - logging.warning( - "Querying from %s to %s took longer than %s!" % ( - old_starttime, - pretty_ts(endtime, rule.get('use_local_time')), - self.run_every - ) + if next_run < datetime.datetime.utcnow(): + # We were processing for longer than our refresh interval + # This can happen if --start was specified with a large time period + # or if we are running too slow to process events in real time. + logging.warning( + "Querying from %s to %s took longer than %s!" % ( + old_starttime, + pretty_ts(endtime, rule.get('use_local_time')), + self.run_every ) + ) - self.remove_old_events(rule) + rule['initial_starttime'] = None - # Only force starttime once - self.starttime = None + self.remove_old_events(rule) - if not self.args.pin_rules: - self.load_rule_changes() + self.reset_rule_schedule(rule) + + def reset_rule_schedule(self, rule): + # We hit the end of a execution schedule, pause ourselves until next run + if rule.get('limit_execution') and rule['next_starttime']: + self.scheduler.modify_job(job_id=rule['name'], next_run_time=rule['next_starttime']) + # If we are preventing covering non-scheduled time periods, reset min_starttime and previous_endtime + if rule['next_min_starttime']: + rule['minimum_starttime'] = rule['next_min_starttime'] + rule['previous_endtime'] = rule['next_min_starttime'] + elastalert_logger.info('Pausing %s until next run at %s' % (rule['name'], pretty_ts(rule['next_starttime']))) def stop(self): """ Stop an ElastAlert runner that's been started """ self.running = False + def get_disabled_rules(self): + """ Return disabled rules """ + return [rule['name'] for rule in self.disabled_rules] + def sleep_for(self, duration): """ Sleep for a set duration """ elastalert_logger.info("Sleeping for %s seconds" % (duration)) @@ -1274,7 +1391,7 @@ def upload_dashboard(self, db, rule, match): # Upload es = elasticsearch_client(rule) - + # TODO: doc_type = _doc for elastic >= 6 res = es.index(index='kibana-int', doc_type='temp', body=db_body) @@ -1293,9 +1410,10 @@ def get_dashboard(self, rule, db_name): raise EAException("use_kibana_dashboard undefined") query = {'query': {'term': {'_id': db_name}}} try: - res = es.search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) + # TODO use doc_type = _doc + res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) except ElasticsearchException as e: - raise EAException("Error querying for dashboard: %s" % (e)), None, sys.exc_info()[2] + raise EAException("Error querying for dashboard: %s" % (e)).with_traceback(sys.exc_info()[2]) if res['hits']['hits']: return json.loads(res['hits']['hits'][0]['_source']['dashboard']) @@ -1350,8 +1468,8 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): # Compute top count keys if rule.get('top_count_keys'): for match in matches: - if 'query_key' in rule and rule['query_key'] in match: - qk = match[rule['query_key']] + if 'query_key' in rule: + qk = lookup_es_key(match, rule['query_key']) else: qk = None @@ -1387,6 +1505,11 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): if kb_link: matches[0]['kibana_link'] = kb_link + if rule.get('generate_kibana_discover_url'): + kb_link = generate_kibana_discover_url(rule, matches[0]) + if kb_link: + matches[0]['kibana_discover_url'] = kb_link + # Enhancements were already run at match time if # run_enhancements_first is set or # retried==True, which means this is a retry of a failed alert @@ -1397,7 +1520,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): try: enhancement.process(match) valid_matches.append(match) - except DropMatchException as e: + except DropMatchException: pass except EAException as e: self.handle_error("Error running match enhancement: %s" % (e), {'rule': rule['name']}) @@ -1425,7 +1548,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): self.handle_error('Error while running alert %s: %s' % (alert.get_info()['type'], e), {'rule': rule['name']}) alert_exception = str(e) else: - self.alerts_sent += 1 + self.thread_data.alerts_sent += 1 alert_sent = True # Write the alert(s) to ES @@ -1435,7 +1558,7 @@ def send_alert(self, matches, rule, alert_time=None, retried=False): # Set all matches to aggregate together if agg_id: alert_body['aggregate_id'] = agg_id - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, rule) if res and not agg_id: agg_id = res['_id'] @@ -1448,6 +1571,15 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No 'alert_time': alert_time } + if rule.get('include_match_in_root'): + body.update({k: v for k, v in match.items() if not k.startswith('_')}) + + if self.add_metadata_alert: + body['category'] = rule['category'] + body['description'] = rule['description'] + body['owner'] = rule['owner'] + body['priority'] = rule['priority'] + match_time = lookup_es_key(match, rule['timestamp_field']) if match_time is not None: body['match_time'] = match_time @@ -1459,18 +1591,14 @@ def get_alert_body(self, match, rule, alert_sent, alert_time, alert_exception=No body['alert_exception'] = alert_exception return body - def writeback(self, doc_type, body): - writeback_index = self.writeback_index - if(self.is_atleastsix()): - writeback_index = self.get_six_index(doc_type) - + def writeback(self, doc_type, body, rule=None, match_body=None): # ES 2.0 - 2.3 does not support dots in field names. if self.replace_dots_in_field_names: writeback_body = replace_dots_in_field_names(body) else: writeback_body = body - for key in writeback_body.keys(): + for key in list(writeback_body.keys()): # Convert any datetime objects to timestamps if isinstance(writeback_body[key], datetime.datetime): writeback_body[key] = dt_to_ts(writeback_body[key]) @@ -1483,8 +1611,11 @@ def writeback(self, doc_type, body): writeback_body['@timestamp'] = dt_to_ts(ts_now()) try: - res = self.writeback_es.index(index=writeback_index, - doc_type=doc_type, body=body) + index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type) + if self.writeback_es.is_atleastsixtwo(): + res = self.writeback_es.index(index=index, body=body) + else: + res = self.writeback_es.index(index=index, doc_type=doc_type, body=body) return res except ElasticsearchException as e: logging.exception("Error writing alert info to Elasticsearch: %s" % (e)) @@ -1501,16 +1632,17 @@ def find_recent_pending_alerts(self, time_limit): time_filter = {'range': {'alert_time': {'from': dt_to_ts(ts_now() - time_limit), 'to': dt_to_ts(ts_now())}}} sort = {'sort': {'alert_time': {'order': 'asc'}}} - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): query = {'query': {'bool': {'must': inner_query, 'filter': time_filter}}} else: query = {'query': inner_query, 'filter': time_filter} query.update(sort) try: - res = self.writeback_es.search(index=self.writeback_index, - doc_type='elastalert', - body=query, - size=1000) + if self.writeback_es.is_atleastsixtwo(): + res = self.writeback_es.search(index=self.writeback_index, body=query, size=1000) + else: + res = self.writeback_es.deprecated_search(index=self.writeback_index, + doc_type='elastalert', body=query, size=1000) if res['hits']['hits']: return res['hits']['hits'] except ElasticsearchException as e: @@ -1539,8 +1671,7 @@ def send_pending_alerts(self): continue # Set current_es for top_count_keys query - self.current_es = elasticsearch_client(rule) - self.current_es_addr = (rule['es_host'], rule['es_port']) + self.thread_data.current_es = elasticsearch_client(rule) # Send the alert unless it's a future alert if ts_now() > ts_to_dt(alert_time): @@ -1556,23 +1687,24 @@ def send_pending_alerts(self): self.alert([match_body], rule, alert_time=alert_time, retried=retried) if rule['current_aggregate_id']: - for qk, agg_id in rule['current_aggregate_id'].iteritems(): + for qk, agg_id in rule['current_aggregate_id'].items(): if agg_id == _id: rule['current_aggregate_id'].pop(qk) break # Delete it from the index try: - self.writeback_es.delete(index=self.writeback_index, - doc_type='elastalert', - id=_id) + if self.writeback_es.is_atleastsixtwo(): + self.writeback_es.delete(index=self.writeback_index, id=_id) + else: + self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=_id) except ElasticsearchException: # TODO: Give this a more relevant exception, try:except: is evil. self.handle_error("Failed to delete alert %s at %s" % (_id, alert_time)) # Send in memory aggregated alerts for rule in self.rules: if rule['agg_matches']: - for aggregation_key_value, aggregate_alert_time in rule['aggregate_alert_time'].iteritems(): + for aggregation_key_value, aggregate_alert_time in rule['aggregate_alert_time'].items(): if ts_now() > aggregate_alert_time: alertable_matches = [ agg_match @@ -1595,15 +1727,18 @@ def get_aggregated_matches(self, _id): query = {'query': {'query_string': {'query': 'aggregate_id:%s' % (_id)}}, 'sort': {'@timestamp': 'asc'}} matches = [] try: - res = self.writeback_es.search(index=self.writeback_index, - doc_type='elastalert', - body=query, - size=self.max_aggregation) + if self.writeback_es.is_atleastsixtwo(): + res = self.writeback_es.search(index=self.writeback_index, body=query, + size=self.max_aggregation) + else: + res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert', + body=query, size=self.max_aggregation) for match in res['hits']['hits']: matches.append(match['_source']) - self.writeback_es.delete(index=self.writeback_index, - doc_type='elastalert', - id=match['_id']) + if self.writeback_es.is_atleastsixtwo(): + self.writeback_es.delete(index=self.writeback_index, id=match['_id']) + else: + self.writeback_es.delete(index=self.writeback_index, doc_type='elastalert', id=match['_id']) except (KeyError, ElasticsearchException) as e: self.handle_error("Error fetching aggregated matches: %s" % (e), {'id': _id}) return matches @@ -1615,14 +1750,14 @@ def find_pending_aggregate_alert(self, rule, aggregation_key_value=None): 'must_not': [{'exists': {'field': 'aggregate_id'}}]}}} if aggregation_key_value: query['filter']['bool']['must'].append({'term': {'aggregation_key': aggregation_key_value}}) - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): query = {'query': {'bool': query}} query['sort'] = {'alert_time': {'order': 'desc'}} try: - res = self.writeback_es.search(index=self.writeback_index, - doc_type='elastalert', - body=query, - size=1) + if self.writeback_es.is_atleastsixtwo(): + res = self.writeback_es.search(index=self.writeback_index, body=query, size=1) + else: + res = self.writeback_es.deprecated_search(index=self.writeback_index, doc_type='elastalert', body=query, size=1) if len(res['hits']['hits']) == 0: return None except (KeyError, ElasticsearchException) as e: @@ -1696,7 +1831,7 @@ def add_aggregated_alert(self, match, rule): alert_body['aggregate_id'] = agg_id if aggregation_key_value: alert_body['aggregation_key'] = aggregation_key_value - res = self.writeback('elastalert', alert_body) + res = self.writeback('elastalert', alert_body, rule) # If new aggregation, save _id if res and not agg_id: @@ -1754,20 +1889,25 @@ def is_silenced(self, rule_name): return False query = {'term': {'rule_name': rule_name}} sort = {'sort': {'until': {'order': 'desc'}}} - if self.is_atleastfive(): + if self.writeback_es.is_atleastfive(): query = {'query': query} else: query = {'filter': query} query.update(sort) try: - if(self.is_atleastsix()): - index = self.get_six_index('silence') - res = self.writeback_es.search(index=index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) + doc_type = 'silence' + index = self.writeback_es.resolve_writeback_index(self.writeback_index, doc_type) + if self.writeback_es.is_atleastsixtwo(): + if self.writeback_es.is_atleastsixsix(): + res = self.writeback_es.search(index=index, size=1, body=query, + _source_includes=['until', 'exponent']) + else: + res = self.writeback_es.search(index=index, size=1, body=query, + _source_include=['until', 'exponent']) else: - res = self.writeback_es.search(index=self.writeback_index, doc_type='silence', - size=1, body=query, _source_include=['until', 'exponent']) + res = self.writeback_es.deprecated_search(index=index, doc_type=doc_type, + size=1, body=query, _source_include=['until', 'exponent']) except ElasticsearchException as e: self.handle_error("Error while querying for alert silence status: %s" % (e), {'rule': rule_name}) @@ -1775,7 +1915,7 @@ def is_silenced(self, rule_name): if res['hits']['hits']: until_ts = res['hits']['hits'][0]['_source']['until'] exponent = res['hits']['hits'][0]['_source'].get('exponent', 0) - if rule_name not in self.silence_cache.keys(): + if rule_name not in list(self.silence_cache.keys()): self.silence_cache[rule_name] = (ts_to_dt(until_ts), exponent) else: self.silence_cache[rule_name] = (ts_to_dt(until_ts), self.silence_cache[rule_name][1]) @@ -1800,6 +1940,7 @@ def handle_uncaught_exception(self, exception, rule): if self.disable_rules_on_error: self.rules = [running_rule for running_rule in self.rules if running_rule['name'] != rule['name']] self.disabled_rules.append(rule) + self.scheduler.pause_job(job_id=rule['name']) elastalert_logger.info('Rule %s disabled', rule['name']) if self.notify_email: self.send_notification_email(exception=exception, rule=rule) @@ -1822,13 +1963,13 @@ def send_notification_email(self, text='', exception=None, rule=None, subject=No tb = traceback.format_exc() email_body += tb - if isinstance(self.notify_email, basestring): + if isinstance(self.notify_email, str): self.notify_email = [self.notify_email] email = MIMEText(email_body) email['Subject'] = subject if subject else 'ElastAlert notification' recipients = self.notify_email if rule and rule.get('notify_email'): - if isinstance(rule['notify_email'], basestring): + if isinstance(rule['notify_email'], str): rule['notify_email'] = [rule['notify_email']] recipients = recipients + rule['notify_email'] recipients = list(set(recipients)) @@ -1855,14 +1996,14 @@ def get_top_counts(self, rule, starttime, endtime, keys, number=None, qk=None): if hits_terms is None: top_events_count = {} else: - buckets = hits_terms.values()[0] + buckets = list(hits_terms.values())[0] # get_hits_terms adds to num_hits, but we don't want to count these - self.num_hits -= len(buckets) + self.thread_data.num_hits -= len(buckets) terms = {} for bucket in buckets: terms[bucket['key']] = bucket['doc_count'] - counts = terms.items() + counts = list(terms.items()) counts.sort(key=lambda x: x[1], reverse=True) top_events_count = dict(counts[:number]) diff --git a/elastalert/enhancements.py b/elastalert/enhancements.py index d6c902514..6cc1cdd57 100644 --- a/elastalert/enhancements.py +++ b/elastalert/enhancements.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from .util import pretty_ts class BaseEnhancement(object): @@ -14,6 +15,11 @@ def process(self, match): raise NotImplementedError() +class TimeEnhancement(BaseEnhancement): + def process(self, match): + match['@timestamp'] = pretty_ts(match['@timestamp']) + + class DropMatchException(Exception): """ ElastAlert will drop a match if this exception type is raised by an enhancement """ pass diff --git a/elastalert/es_mappings/5/elastalert.json b/elastalert/es_mappings/5/elastalert.json new file mode 100644 index 000000000..b522933b3 --- /dev/null +++ b/elastalert/es_mappings/5/elastalert.json @@ -0,0 +1,30 @@ +{ + "elastalert": { + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "alert_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_body": { + "type": "object", + "enabled": "false" + }, + "aggregate_id": { + "index": "not_analyzed", + "type": "string" + } + } + } +} diff --git a/elastalert/es_mappings/5/elastalert_error.json b/elastalert/es_mappings/5/elastalert_error.json new file mode 100644 index 000000000..7f1b3c0a8 --- /dev/null +++ b/elastalert/es_mappings/5/elastalert_error.json @@ -0,0 +1,14 @@ +{ + "elastalert_error": { + "properties": { + "data": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} diff --git a/elastalert/es_mappings/5/elastalert_status.json b/elastalert/es_mappings/5/elastalert_status.json new file mode 100644 index 000000000..f8cd9643f --- /dev/null +++ b/elastalert/es_mappings/5/elastalert_status.json @@ -0,0 +1,14 @@ +{ + "elastalert_status": { + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} diff --git a/elastalert/es_mappings/5/past_elastalert.json b/elastalert/es_mappings/5/past_elastalert.json new file mode 100644 index 000000000..e10783748 --- /dev/null +++ b/elastalert/es_mappings/5/past_elastalert.json @@ -0,0 +1,22 @@ +{ + "past_elastalert": { + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "match_body": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "aggregate_id": { + "index": "not_analyzed", + "type": "string" + } + } + } +} diff --git a/elastalert/es_mappings/5/silence.json b/elastalert/es_mappings/5/silence.json new file mode 100644 index 000000000..b04006da8 --- /dev/null +++ b/elastalert/es_mappings/5/silence.json @@ -0,0 +1,18 @@ +{ + "silence": { + "properties": { + "rule_name": { + "index": "not_analyzed", + "type": "string" + }, + "until": { + "type": "date", + "format": "dateOptionalTime" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } + } +} diff --git a/elastalert/es_mappings/6/elastalert.json b/elastalert/es_mappings/6/elastalert.json new file mode 100644 index 000000000..645a67762 --- /dev/null +++ b/elastalert/es_mappings/6/elastalert.json @@ -0,0 +1,38 @@ +{ + "numeric_detection": true, + "date_detection": false, + "dynamic_templates": [ + { + "strings_as_keyword": { + "mapping": { + "ignore_above": 1024, + "type": "keyword" + }, + "match_mapping_type": "string" + } + } + ], + "properties": { + "rule_name": { + "type": "keyword" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "alert_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_time": { + "type": "date", + "format": "dateOptionalTime" + }, + "match_body": { + "type": "object" + }, + "aggregate_id": { + "type": "keyword" + } + } +} diff --git a/elastalert/es_mappings/6/elastalert_error.json b/elastalert/es_mappings/6/elastalert_error.json new file mode 100644 index 000000000..b4b577c16 --- /dev/null +++ b/elastalert/es_mappings/6/elastalert_error.json @@ -0,0 +1,12 @@ +{ + "properties": { + "data": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } +} diff --git a/elastalert/es_mappings/6/elastalert_status.json b/elastalert/es_mappings/6/elastalert_status.json new file mode 100644 index 000000000..eea1762af --- /dev/null +++ b/elastalert/es_mappings/6/elastalert_status.json @@ -0,0 +1,11 @@ +{ + "properties": { + "rule_name": { + "type": "keyword" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } +} diff --git a/elastalert/es_mappings/6/past_elastalert.json b/elastalert/es_mappings/6/past_elastalert.json new file mode 100644 index 000000000..0cf2c67db --- /dev/null +++ b/elastalert/es_mappings/6/past_elastalert.json @@ -0,0 +1,18 @@ +{ + "properties": { + "rule_name": { + "type": "keyword" + }, + "match_body": { + "type": "object", + "enabled": "false" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + }, + "aggregate_id": { + "type": "keyword" + } + } +} diff --git a/elastalert/es_mappings/6/silence.json b/elastalert/es_mappings/6/silence.json new file mode 100644 index 000000000..d30c2b066 --- /dev/null +++ b/elastalert/es_mappings/6/silence.json @@ -0,0 +1,15 @@ +{ + "properties": { + "rule_name": { + "type": "keyword" + }, + "until": { + "type": "date", + "format": "dateOptionalTime" + }, + "@timestamp": { + "type": "date", + "format": "dateOptionalTime" + } + } +} diff --git a/elastalert/kibana.py b/elastalert/kibana.py index 2cd557bff..de690494e 100644 --- a/elastalert/kibana.py +++ b/elastalert/kibana.py @@ -1,173 +1,176 @@ # -*- coding: utf-8 -*- +# flake8: noqa import os.path -import urllib +import urllib.error +import urllib.parse +import urllib.request -from util import EAException +from .util import EAException dashboard_temp = {'editable': True, - u'failover': False, - u'index': {u'default': u'NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED', - u'interval': u'none', - u'pattern': u'', - u'warm_fields': True}, - u'loader': {u'hide': False, - u'load_elasticsearch': True, - u'load_elasticsearch_size': 20, - u'load_gist': True, - u'load_local': True, - u'save_default': True, - u'save_elasticsearch': True, - u'save_gist': False, - u'save_local': True, - u'save_temp': True, - u'save_temp_ttl': u'30d', - u'save_temp_ttl_enable': True}, - u'nav': [{u'collapse': False, - u'enable': True, - u'filter_id': 0, - u'notice': False, - u'now': False, - u'refresh_intervals': [u'5s', - u'10s', - u'30s', - u'1m', - u'5m', - u'15m', - u'30m', - u'1h', - u'2h', - u'1d'], - u'status': u'Stable', - u'time_options': [u'5m', - u'15m', - u'1h', - u'6h', - u'12h', - u'24h', - u'2d', - u'7d', - u'30d'], - u'timefield': u'@timestamp', - u'type': u'timepicker'}], - u'panel_hints': True, - u'pulldowns': [{u'collapse': False, - u'enable': True, - u'notice': True, - u'type': u'filtering'}], - u'refresh': False, - u'rows': [{u'collapsable': True, - u'collapse': False, - u'editable': True, - u'height': u'350px', - u'notice': False, - u'panels': [{u'annotate': {u'enable': False, - u'field': u'_type', - u'query': u'*', - u'size': 20, - u'sort': [u'_score', u'desc']}, - u'auto_int': True, - u'bars': True, - u'derivative': False, - u'editable': True, - u'fill': 3, - u'grid': {u'max': None, u'min': 0}, - u'group': [u'default'], - u'interactive': True, - u'interval': u'1m', - u'intervals': [u'auto', - u'1s', - u'1m', - u'5m', - u'10m', - u'30m', - u'1h', - u'3h', - u'12h', - u'1d', - u'1w', - u'1M', - u'1y'], - u'legend': True, - u'legend_counts': True, - u'lines': False, - u'linewidth': 3, - u'mode': u'count', - u'options': True, - u'percentage': False, - u'pointradius': 5, - u'points': False, - u'queries': {u'ids': [0], u'mode': u'all'}, - u'resolution': 100, - u'scale': 1, - u'show_query': True, - u'span': 12, - u'spyable': True, - u'stack': True, - u'time_field': u'@timestamp', - u'timezone': u'browser', - u'title': u'Events over time', - u'tooltip': {u'query_as_alias': True, - u'value_type': u'cumulative'}, - u'type': u'histogram', - u'value_field': None, - u'x-axis': True, - u'y-axis': True, - u'y_format': u'none', - u'zerofill': True, - u'zoomlinks': True}], - u'title': u'Graph'}, - {u'collapsable': True, - u'collapse': False, - u'editable': True, - u'height': u'350px', - u'notice': False, - u'panels': [{u'all_fields': False, - u'editable': True, - u'error': False, - u'field_list': True, - u'fields': [], - u'group': [u'default'], - u'header': True, - u'highlight': [], - u'localTime': True, - u'normTimes': True, - u'offset': 0, - u'overflow': u'min-height', - u'pages': 5, - u'paging': True, - u'queries': {u'ids': [0], u'mode': u'all'}, - u'size': 100, - u'sort': [u'@timestamp', u'desc'], - u'sortable': True, - u'span': 12, - u'spyable': True, - u'status': u'Stable', - u'style': {u'font-size': u'9pt'}, - u'timeField': u'@timestamp', - u'title': u'All events', - u'trimFactor': 300, - u'type': u'table'}], - u'title': u'Events'}], - u'services': {u'filter': {u'ids': [0], - u'list': {u'0': {u'active': True, - u'alias': u'', - u'field': u'@timestamp', - u'from': u'now-24h', - u'id': 0, - u'mandate': u'must', - u'to': u'now', - u'type': u'time'}}}, - u'query': {u'ids': [0], - u'list': {u'0': {u'alias': u'', - u'color': u'#7EB26D', - u'enable': True, - u'id': 0, - u'pin': False, - u'query': u'', - u'type': u'lucene'}}}}, - u'style': u'dark', - u'title': u'ElastAlert Alert Dashboard'} + 'failover': False, + 'index': {'default': 'NO_TIME_FILTER_OR_INDEX_PATTERN_NOT_MATCHED', + 'interval': 'none', + 'pattern': '', + 'warm_fields': True}, + 'loader': {'hide': False, + 'load_elasticsearch': True, + 'load_elasticsearch_size': 20, + 'load_gist': True, + 'load_local': True, + 'save_default': True, + 'save_elasticsearch': True, + 'save_gist': False, + 'save_local': True, + 'save_temp': True, + 'save_temp_ttl': '30d', + 'save_temp_ttl_enable': True}, + 'nav': [{'collapse': False, + 'enable': True, + 'filter_id': 0, + 'notice': False, + 'now': False, + 'refresh_intervals': ['5s', + '10s', + '30s', + '1m', + '5m', + '15m', + '30m', + '1h', + '2h', + '1d'], + 'status': 'Stable', + 'time_options': ['5m', + '15m', + '1h', + '6h', + '12h', + '24h', + '2d', + '7d', + '30d'], + 'timefield': '@timestamp', + 'type': 'timepicker'}], + 'panel_hints': True, + 'pulldowns': [{'collapse': False, + 'enable': True, + 'notice': True, + 'type': 'filtering'}], + 'refresh': False, + 'rows': [{'collapsable': True, + 'collapse': False, + 'editable': True, + 'height': '350px', + 'notice': False, + 'panels': [{'annotate': {'enable': False, + 'field': '_type', + 'query': '*', + 'size': 20, + 'sort': ['_score', 'desc']}, + 'auto_int': True, + 'bars': True, + 'derivative': False, + 'editable': True, + 'fill': 3, + 'grid': {'max': None, 'min': 0}, + 'group': ['default'], + 'interactive': True, + 'interval': '1m', + 'intervals': ['auto', + '1s', + '1m', + '5m', + '10m', + '30m', + '1h', + '3h', + '12h', + '1d', + '1w', + '1M', + '1y'], + 'legend': True, + 'legend_counts': True, + 'lines': False, + 'linewidth': 3, + 'mode': 'count', + 'options': True, + 'percentage': False, + 'pointradius': 5, + 'points': False, + 'queries': {'ids': [0], 'mode': 'all'}, + 'resolution': 100, + 'scale': 1, + 'show_query': True, + 'span': 12, + 'spyable': True, + 'stack': True, + 'time_field': '@timestamp', + 'timezone': 'browser', + 'title': 'Events over time', + 'tooltip': {'query_as_alias': True, + 'value_type': 'cumulative'}, + 'type': 'histogram', + 'value_field': None, + 'x-axis': True, + 'y-axis': True, + 'y_format': 'none', + 'zerofill': True, + 'zoomlinks': True}], + 'title': 'Graph'}, + {'collapsable': True, + 'collapse': False, + 'editable': True, + 'height': '350px', + 'notice': False, + 'panels': [{'all_fields': False, + 'editable': True, + 'error': False, + 'field_list': True, + 'fields': [], + 'group': ['default'], + 'header': True, + 'highlight': [], + 'localTime': True, + 'normTimes': True, + 'offset': 0, + 'overflow': 'min-height', + 'pages': 5, + 'paging': True, + 'queries': {'ids': [0], 'mode': 'all'}, + 'size': 100, + 'sort': ['@timestamp', 'desc'], + 'sortable': True, + 'span': 12, + 'spyable': True, + 'status': 'Stable', + 'style': {'font-size': '9pt'}, + 'timeField': '@timestamp', + 'title': 'All events', + 'trimFactor': 300, + 'type': 'table'}], + 'title': 'Events'}], + 'services': {'filter': {'ids': [0], + 'list': {'0': {'active': True, + 'alias': '', + 'field': '@timestamp', + 'from': 'now-24h', + 'id': 0, + 'mandate': 'must', + 'to': 'now', + 'type': 'time'}}}, + 'query': {'ids': [0], + 'list': {'0': {'alias': '', + 'color': '#7EB26D', + 'enable': True, + 'id': 0, + 'pin': False, + 'query': '', + 'type': 'lucene'}}}}, + 'style': 'dark', + 'title': 'ElastAlert Alert Dashboard'} kibana4_time_temp = "(refreshInterval:(display:Off,section:0,value:0),time:(from:'%s',mode:absolute,to:'%s'))" @@ -213,9 +216,9 @@ def add_filter(dashboard, es_filter): kibana_filter['query'] = es_filter['query_string']['query'] elif 'term' in es_filter: kibana_filter['type'] = 'field' - f_field, f_query = es_filter['term'].items()[0] + f_field, f_query = list(es_filter['term'].items())[0] # Wrap query in quotes, otherwise certain characters cause Kibana to throw errors - if isinstance(f_query, basestring): + if isinstance(f_query, str): f_query = '"%s"' % (f_query.replace('"', '\\"')) if isinstance(f_query, list): # Escape quotes @@ -228,7 +231,7 @@ def add_filter(dashboard, es_filter): kibana_filter['query'] = f_query elif 'range' in es_filter: kibana_filter['type'] = 'range' - f_field, f_range = es_filter['range'].items()[0] + f_field, f_range = list(es_filter['range'].items())[0] kibana_filter['field'] = f_field kibana_filter.update(f_range) else: @@ -250,7 +253,7 @@ def filters_from_dashboard(db): filters = db['services']['filter']['list'] config_filters = [] or_filters = [] - for filter in filters.values(): + for filter in list(filters.values()): filter_type = filter['type'] if filter_type == 'time': continue @@ -281,5 +284,5 @@ def filters_from_dashboard(db): def kibana4_dashboard_link(dashboard, starttime, endtime): dashboard = os.path.expandvars(dashboard) time_settings = kibana4_time_temp % (starttime, endtime) - time_settings = urllib.quote(time_settings) + time_settings = urllib.parse.quote(time_settings) return "%s?_g=%s" % (dashboard, time_settings) diff --git a/elastalert/kibana_discover.py b/elastalert/kibana_discover.py new file mode 100644 index 000000000..7e4dbb5d1 --- /dev/null +++ b/elastalert/kibana_discover.py @@ -0,0 +1,192 @@ +# -*- coding: utf-8 -*- +# flake8: noqa +import datetime +import logging +import json +import os.path +import prison +import urllib.parse + +from .util import EAException +from .util import lookup_es_key +from .util import ts_add + +kibana_default_timedelta = datetime.timedelta(minutes=10) + +kibana5_kibana6_versions = frozenset(['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) +kibana7_versions = frozenset(['7.0', '7.1', '7.2', '7.3']) + +def generate_kibana_discover_url(rule, match): + ''' Creates a link for a kibana discover app. ''' + + discover_app_url = rule.get('kibana_discover_app_url') + if not discover_app_url: + logging.warning( + 'Missing kibana_discover_app_url for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + kibana_version = rule.get('kibana_discover_version') + if not kibana_version: + logging.warning( + 'Missing kibana_discover_version for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + index = rule.get('kibana_discover_index_pattern_id') + if not index: + logging.warning( + 'Missing kibana_discover_index_pattern_id for rule %s' % ( + rule.get('name', '') + ) + ) + return None + + columns = rule.get('kibana_discover_columns', ['_source']) + filters = rule.get('filter', []) + + if 'query_key' in rule: + query_keys = rule.get('compound_query_key', [rule['query_key']]) + else: + query_keys = [] + + timestamp = lookup_es_key(match, rule['timestamp_field']) + timeframe = rule.get('timeframe', kibana_default_timedelta) + from_timedelta = rule.get('kibana_discover_from_timedelta', timeframe) + from_time = ts_add(timestamp, -from_timedelta) + to_timedelta = rule.get('kibana_discover_to_timedelta', timeframe) + to_time = ts_add(timestamp, to_timedelta) + + if kibana_version in kibana5_kibana6_versions: + globalState = kibana6_disover_global_state(from_time, to_time) + appState = kibana_discover_app_state(index, columns, filters, query_keys, match) + + elif kibana_version in kibana7_versions: + globalState = kibana7_disover_global_state(from_time, to_time) + appState = kibana_discover_app_state(index, columns, filters, query_keys, match) + + else: + logging.warning( + 'Unknown kibana discover application version %s for rule %s' % ( + kibana_version, + rule.get('name', '') + ) + ) + return None + + return "%s?_g=%s&_a=%s" % ( + os.path.expandvars(discover_app_url), + urllib.parse.quote(globalState), + urllib.parse.quote(appState) + ) + + +def kibana6_disover_global_state(from_time, to_time): + return prison.dumps( { + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'mode': 'absolute', + 'to': to_time + } + } ) + + +def kibana7_disover_global_state(from_time, to_time): + return prison.dumps( { + 'filters': [], + 'refreshInterval': { + 'pause': True, + 'value': 0 + }, + 'time': { + 'from': from_time, + 'to': to_time + } + } ) + + +def kibana_discover_app_state(index, columns, filters, query_keys, match): + app_filters = [] + + if filters: + bool_filter = { 'must': filters } + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'bool': bool_filter, + 'meta': { + 'alias': 'filter', + 'disabled': False, + 'index': index, + 'key': 'bool', + 'negate': False, + 'type': 'custom', + 'value': json.dumps(bool_filter, separators=(',', ':')) + }, + } ) + + for query_key in query_keys: + query_value = lookup_es_key(match, query_key) + + if query_value is None: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'exists': { + 'field': query_key + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': True, + 'type': 'exists', + 'value': 'exists' + } + } ) + + else: + app_filters.append( { + '$state': { + 'store': 'appState' + }, + 'meta': { + 'alias': None, + 'disabled': False, + 'index': index, + 'key': query_key, + 'negate': False, + 'params': { + 'query': query_value, + 'type': 'phrase' + }, + 'type': 'phrase', + 'value': str(query_value) + }, + 'query': { + 'match': { + query_key: { + 'query': query_value, + 'type': 'phrase' + } + } + } + } ) + + return prison.dumps( { + 'columns': columns, + 'filters': app_filters, + 'index': index, + 'interval': 'auto' + } ) diff --git a/elastalert/loaders.py b/elastalert/loaders.py new file mode 100644 index 000000000..771194768 --- /dev/null +++ b/elastalert/loaders.py @@ -0,0 +1,553 @@ +# -*- coding: utf-8 -*- +import copy +import datetime +import hashlib +import logging +import os +import sys + +import jsonschema +import yaml +import yaml.scanner +from staticconf.loader import yaml_loader + +from . import alerts +from . import enhancements +from . import ruletypes +from .opsgenie import OpsGenieAlerter +from .util import dt_to_ts +from .util import dt_to_ts_with_format +from .util import dt_to_unix +from .util import dt_to_unixms +from .util import EAException +from .util import get_module +from .util import ts_to_dt +from .util import ts_to_dt_with_format +from .util import unix_to_dt +from .util import unixms_to_dt + + +class RulesLoader(object): + # import rule dependency + import_rules = {} + + # Required global (config.yaml) configuration options for the loader + required_globals = frozenset([]) + + # Required local (rule.yaml) configuration options + required_locals = frozenset(['alert', 'type', 'name', 'index']) + + # Used to map the names of rules to their classes + rules_mapping = { + 'frequency': ruletypes.FrequencyRule, + 'any': ruletypes.AnyRule, + 'spike': ruletypes.SpikeRule, + 'blacklist': ruletypes.BlacklistRule, + 'whitelist': ruletypes.WhitelistRule, + 'change': ruletypes.ChangeRule, + 'flatline': ruletypes.FlatlineRule, + 'new_term': ruletypes.NewTermsRule, + 'cardinality': ruletypes.CardinalityRule, + 'metric_aggregation': ruletypes.MetricAggregationRule, + 'percentage_match': ruletypes.PercentageMatchRule, + 'spike_aggregation': ruletypes.SpikeMetricAggregationRule, + } + + # Used to map names of alerts to their classes + alerts_mapping = { + 'email': alerts.EmailAlerter, + 'jira': alerts.JiraAlerter, + 'opsgenie': OpsGenieAlerter, + 'stomp': alerts.StompAlerter, + 'debug': alerts.DebugAlerter, + 'command': alerts.CommandAlerter, + 'sns': alerts.SnsAlerter, + 'hipchat': alerts.HipChatAlerter, + 'stride': alerts.StrideAlerter, + 'ms_teams': alerts.MsTeamsAlerter, + 'slack': alerts.SlackAlerter, + 'mattermost': alerts.MattermostAlerter, + 'pagerduty': alerts.PagerDutyAlerter, + 'exotel': alerts.ExotelAlerter, + 'twilio': alerts.TwilioAlerter, + 'victorops': alerts.VictorOpsAlerter, + 'telegram': alerts.TelegramAlerter, + 'googlechat': alerts.GoogleChatAlerter, + 'gitter': alerts.GitterAlerter, + 'servicenow': alerts.ServiceNowAlerter, + 'alerta': alerts.AlertaAlerter, + 'post': alerts.HTTPPostAlerter, + 'hivealerter': alerts.HiveAlerter + } + + # A partial ordering of alert types. Relative order will be preserved in the resulting alerts list + # For example, jira goes before email so the ticket # will be added to the resulting email. + alerts_order = { + 'jira': 0, + 'email': 1 + } + + base_config = {} + + def __init__(self, conf): + # schema for rule yaml + self.rule_schema = jsonschema.Draft7Validator( + yaml.load(open(os.path.join(os.path.dirname(__file__), 'schema.yaml')), Loader=yaml.FullLoader)) + + self.base_config = copy.deepcopy(conf) + + def load(self, conf, args=None): + """ + Discover and load all the rules as defined in the conf and args. + :param dict conf: Configuration dict + :param dict args: Arguments dict + :return: List of rules + :rtype: list + """ + names = [] + use_rule = None if args is None else args.rule + + # Load each rule configuration file + rules = [] + rule_files = self.get_names(conf, use_rule) + for rule_file in rule_files: + try: + rule = self.load_configuration(rule_file, conf, args) + # A rule failed to load, don't try to process it + if not rule: + logging.error('Invalid rule file skipped: %s' % rule_file) + continue + # By setting "is_enabled: False" in rule file, a rule is easily disabled + if 'is_enabled' in rule and not rule['is_enabled']: + continue + if rule['name'] in names: + raise EAException('Duplicate rule named %s' % (rule['name'])) + except EAException as e: + raise EAException('Error loading file %s: %s' % (rule_file, e)) + + rules.append(rule) + names.append(rule['name']) + + return rules + + def get_names(self, conf, use_rule=None): + """ + Return a list of rule names that can be passed to `get_yaml` to retrieve. + :param dict conf: Configuration dict + :param str use_rule: Limit to only specified rule + :return: A list of rule names + :rtype: list + """ + raise NotImplementedError() + + def get_hashes(self, conf, use_rule=None): + """ + Discover and get the hashes of all the rules as defined in the conf. + :param dict conf: Configuration + :param str use_rule: Limit to only specified rule + :return: Dict of rule name to hash + :rtype: dict + """ + raise NotImplementedError() + + def get_yaml(self, filename): + """ + Get and parse the yaml of the specified rule. + :param str filename: Rule to get the yaml + :return: Rule YAML dict + :rtype: dict + """ + raise NotImplementedError() + + def get_import_rule(self, rule): + """ + Retrieve the name of the rule to import. + :param dict rule: Rule dict + :return: rule name that will all `get_yaml` to retrieve the yaml of the rule + :rtype: str + """ + return rule['import'] + + def load_configuration(self, filename, conf, args=None): + """ Load a yaml rule file and fill in the relevant fields with objects. + + :param str filename: The name of a rule configuration file. + :param dict conf: The global configuration dictionary, used for populating defaults. + :param dict args: Arguments + :return: The rule configuration, a dictionary. + """ + rule = self.load_yaml(filename) + self.load_options(rule, conf, filename, args) + self.load_modules(rule, args) + return rule + + def load_yaml(self, filename): + """ + Load the rule including all dependency rules. + :param str filename: Rule to load + :return: Loaded rule dict + :rtype: dict + """ + rule = { + 'rule_file': filename, + } + + self.import_rules.pop(filename, None) # clear `filename` dependency + while True: + loaded = self.get_yaml(filename) + + # Special case for merging filters - if both files specify a filter merge (AND) them + if 'filter' in rule and 'filter' in loaded: + rule['filter'] = loaded['filter'] + rule['filter'] + + loaded.update(rule) + rule = loaded + if 'import' in rule: + # Find the path of the next file. + import_filename = self.get_import_rule(rule) + # set dependencies + rules = self.import_rules.get(filename, []) + rules.append(import_filename) + self.import_rules[filename] = rules + filename = import_filename + del (rule['import']) # or we could go on forever! + else: + break + + return rule + + def load_options(self, rule, conf, filename, args=None): + """ Converts time objects, sets defaults, and validates some settings. + + :param rule: A dictionary of parsed YAML from a rule config file. + :param conf: The global configuration dictionary, used for populating defaults. + :param filename: Name of the rule + :param args: Arguments + """ + self.adjust_deprecated_values(rule) + + try: + self.rule_schema.validate(rule) + except jsonschema.ValidationError as e: + raise EAException("Invalid Rule file: %s\n%s" % (filename, e)) + + try: + # Set all time based parameters + if 'timeframe' in rule: + rule['timeframe'] = datetime.timedelta(**rule['timeframe']) + if 'realert' in rule: + rule['realert'] = datetime.timedelta(**rule['realert']) + else: + if 'aggregation' in rule: + rule['realert'] = datetime.timedelta(minutes=0) + else: + rule['realert'] = datetime.timedelta(minutes=1) + if 'aggregation' in rule and not rule['aggregation'].get('schedule'): + rule['aggregation'] = datetime.timedelta(**rule['aggregation']) + if 'query_delay' in rule: + rule['query_delay'] = datetime.timedelta(**rule['query_delay']) + if 'buffer_time' in rule: + rule['buffer_time'] = datetime.timedelta(**rule['buffer_time']) + if 'run_every' in rule: + rule['run_every'] = datetime.timedelta(**rule['run_every']) + if 'bucket_interval' in rule: + rule['bucket_interval_timedelta'] = datetime.timedelta(**rule['bucket_interval']) + if 'exponential_realert' in rule: + rule['exponential_realert'] = datetime.timedelta(**rule['exponential_realert']) + if 'kibana4_start_timedelta' in rule: + rule['kibana4_start_timedelta'] = datetime.timedelta(**rule['kibana4_start_timedelta']) + if 'kibana4_end_timedelta' in rule: + rule['kibana4_end_timedelta'] = datetime.timedelta(**rule['kibana4_end_timedelta']) + if 'kibana_discover_from_timedelta' in rule: + rule['kibana_discover_from_timedelta'] = datetime.timedelta(**rule['kibana_discover_from_timedelta']) + if 'kibana_discover_to_timedelta' in rule: + rule['kibana_discover_to_timedelta'] = datetime.timedelta(**rule['kibana_discover_to_timedelta']) + except (KeyError, TypeError) as e: + raise EAException('Invalid time format used: %s' % e) + + # Set defaults, copy defaults from config.yaml + for key, val in list(self.base_config.items()): + rule.setdefault(key, val) + rule.setdefault('name', os.path.splitext(filename)[0]) + rule.setdefault('realert', datetime.timedelta(seconds=0)) + rule.setdefault('aggregation', datetime.timedelta(seconds=0)) + rule.setdefault('query_delay', datetime.timedelta(seconds=0)) + rule.setdefault('timestamp_field', '@timestamp') + rule.setdefault('filter', []) + rule.setdefault('timestamp_type', 'iso') + rule.setdefault('timestamp_format', '%Y-%m-%dT%H:%M:%SZ') + rule.setdefault('_source_enabled', True) + rule.setdefault('use_local_time', True) + rule.setdefault('description', "") + + # Set timestamp_type conversion function, used when generating queries and processing hits + rule['timestamp_type'] = rule['timestamp_type'].strip().lower() + if rule['timestamp_type'] == 'iso': + rule['ts_to_dt'] = ts_to_dt + rule['dt_to_ts'] = dt_to_ts + elif rule['timestamp_type'] == 'unix': + rule['ts_to_dt'] = unix_to_dt + rule['dt_to_ts'] = dt_to_unix + elif rule['timestamp_type'] == 'unix_ms': + rule['ts_to_dt'] = unixms_to_dt + rule['dt_to_ts'] = dt_to_unixms + elif rule['timestamp_type'] == 'custom': + def _ts_to_dt_with_format(ts): + return ts_to_dt_with_format(ts, ts_format=rule['timestamp_format']) + + def _dt_to_ts_with_format(dt): + ts = dt_to_ts_with_format(dt, ts_format=rule['timestamp_format']) + if 'timestamp_format_expr' in rule: + # eval expression passing 'ts' and 'dt' + return eval(rule['timestamp_format_expr'], {'ts': ts, 'dt': dt}) + else: + return ts + + rule['ts_to_dt'] = _ts_to_dt_with_format + rule['dt_to_ts'] = _dt_to_ts_with_format + else: + raise EAException('timestamp_type must be one of iso, unix, or unix_ms') + + # Add support for client ssl certificate auth + if 'verify_certs' in conf: + rule.setdefault('verify_certs', conf.get('verify_certs')) + rule.setdefault('ca_certs', conf.get('ca_certs')) + rule.setdefault('client_cert', conf.get('client_cert')) + rule.setdefault('client_key', conf.get('client_key')) + + # Set HipChat options from global config + rule.setdefault('hipchat_msg_color', 'red') + rule.setdefault('hipchat_domain', 'api.hipchat.com') + rule.setdefault('hipchat_notify', True) + rule.setdefault('hipchat_from', '') + rule.setdefault('hipchat_ignore_ssl_errors', False) + + # Make sure we have required options + if self.required_locals - frozenset(list(rule.keys())): + raise EAException('Missing required option(s): %s' % (', '.join(self.required_locals - frozenset(list(rule.keys()))))) + + if 'include' in rule and type(rule['include']) != list: + raise EAException('include option must be a list') + + raw_query_key = rule.get('query_key') + if isinstance(raw_query_key, list): + if len(raw_query_key) > 1: + rule['compound_query_key'] = raw_query_key + rule['query_key'] = ','.join(raw_query_key) + elif len(raw_query_key) == 1: + rule['query_key'] = raw_query_key[0] + else: + del(rule['query_key']) + + if isinstance(rule.get('aggregation_key'), list): + rule['compound_aggregation_key'] = rule['aggregation_key'] + rule['aggregation_key'] = ','.join(rule['aggregation_key']) + + if isinstance(rule.get('compare_key'), list): + rule['compound_compare_key'] = rule['compare_key'] + rule['compare_key'] = ','.join(rule['compare_key']) + elif 'compare_key' in rule: + rule['compound_compare_key'] = [rule['compare_key']] + # Add QK, CK and timestamp to include + include = rule.get('include', ['*']) + if 'query_key' in rule: + include.append(rule['query_key']) + if 'compound_query_key' in rule: + include += rule['compound_query_key'] + if 'compound_aggregation_key' in rule: + include += rule['compound_aggregation_key'] + if 'compare_key' in rule: + include.append(rule['compare_key']) + if 'compound_compare_key' in rule: + include += rule['compound_compare_key'] + if 'top_count_keys' in rule: + include += rule['top_count_keys'] + include.append(rule['timestamp_field']) + rule['include'] = list(set(include)) + + # Check that generate_kibana_url is compatible with the filters + if rule.get('generate_kibana_link'): + for es_filter in rule.get('filter'): + if es_filter: + if 'not' in es_filter: + es_filter = es_filter['not'] + if 'query' in es_filter: + es_filter = es_filter['query'] + if list(es_filter.keys())[0] not in ('term', 'query_string', 'range'): + raise EAException( + 'generate_kibana_link is incompatible with filters other than term, query_string and range.' + 'Consider creating a dashboard and using use_kibana_dashboard instead.') + + # Check that doc_type is provided if use_count/terms_query + if rule.get('use_count_query') or rule.get('use_terms_query'): + if 'doc_type' not in rule: + raise EAException('doc_type must be specified.') + + # Check that query_key is set if use_terms_query + if rule.get('use_terms_query'): + if 'query_key' not in rule: + raise EAException('query_key must be specified with use_terms_query') + + # Warn if use_strf_index is used with %y, %M or %D + # (%y = short year, %M = minutes, %D = full date) + if rule.get('use_strftime_index'): + for token in ['%y', '%M', '%D']: + if token in rule.get('index'): + logging.warning('Did you mean to use %s in the index? ' + 'The index will be formatted like %s' % (token, + datetime.datetime.now().strftime( + rule.get('index')))) + + if rule.get('scan_entire_timeframe') and not rule.get('timeframe'): + raise EAException('scan_entire_timeframe can only be used if there is a timeframe specified') + + def load_modules(self, rule, args=None): + """ Loads things that could be modules. Enhancements, alerts and rule type. """ + # Set match enhancements + match_enhancements = [] + for enhancement_name in rule.get('match_enhancements', []): + if enhancement_name in dir(enhancements): + enhancement = getattr(enhancements, enhancement_name) + else: + enhancement = get_module(enhancement_name) + if not issubclass(enhancement, enhancements.BaseEnhancement): + raise EAException("Enhancement module %s not a subclass of BaseEnhancement" % enhancement_name) + match_enhancements.append(enhancement(rule)) + rule['match_enhancements'] = match_enhancements + + # Convert rule type into RuleType object + if rule['type'] in self.rules_mapping: + rule['type'] = self.rules_mapping[rule['type']] + else: + rule['type'] = get_module(rule['type']) + if not issubclass(rule['type'], ruletypes.RuleType): + raise EAException('Rule module %s is not a subclass of RuleType' % (rule['type'])) + + # Make sure we have required alert and type options + reqs = rule['type'].required_options + + if reqs - frozenset(list(rule.keys())): + raise EAException('Missing required option(s): %s' % (', '.join(reqs - frozenset(list(rule.keys()))))) + # Instantiate rule + try: + rule['type'] = rule['type'](rule, args) + except (KeyError, EAException) as e: + raise EAException('Error initializing rule %s: %s' % (rule['name'], e)).with_traceback(sys.exc_info()[2]) + # Instantiate alerts only if we're not in debug mode + # In debug mode alerts are not actually sent so don't bother instantiating them + if not args or not args.debug: + rule['alert'] = self.load_alerts(rule, alert_field=rule['alert']) + + def load_alerts(self, rule, alert_field): + def normalize_config(alert): + """Alert config entries are either "alertType" or {"alertType": {"key": "data"}}. + This function normalizes them both to the latter format. """ + if isinstance(alert, str): + return alert, rule + elif isinstance(alert, dict): + name, config = next(iter(list(alert.items()))) + config_copy = copy.copy(rule) + config_copy.update(config) # warning, this (intentionally) mutates the rule dict + return name, config_copy + else: + raise EAException() + + def create_alert(alert, alert_config): + alert_class = self.alerts_mapping.get(alert) or get_module(alert) + if not issubclass(alert_class, alerts.Alerter): + raise EAException('Alert module %s is not a subclass of Alerter' % alert) + missing_options = (rule['type'].required_options | alert_class.required_options) - frozenset( + alert_config or []) + if missing_options: + raise EAException('Missing required option(s): %s' % (', '.join(missing_options))) + return alert_class(alert_config) + + try: + if type(alert_field) != list: + alert_field = [alert_field] + + alert_field = [normalize_config(x) for x in alert_field] + alert_field = sorted(alert_field, key=lambda a_b: self.alerts_order.get(a_b[0], 1)) + # Convert all alerts into Alerter objects + alert_field = [create_alert(a, b) for a, b in alert_field] + + except (KeyError, EAException) as e: + raise EAException('Error initiating alert %s: %s' % (rule['alert'], e)).with_traceback(sys.exc_info()[2]) + + return alert_field + + @staticmethod + def adjust_deprecated_values(rule): + # From rename of simple HTTP alerter + if rule.get('type') == 'simple': + rule['type'] = 'post' + if 'simple_proxy' in rule: + rule['http_post_proxy'] = rule['simple_proxy'] + if 'simple_webhook_url' in rule: + rule['http_post_url'] = rule['simple_webhook_url'] + logging.warning( + '"simple" alerter has been renamed "post" and comptability may be removed in a future release.') + + +class FileRulesLoader(RulesLoader): + + # Required global (config.yaml) configuration options for the loader + required_globals = frozenset(['rules_folder']) + + def get_names(self, conf, use_rule=None): + # Passing a filename directly can bypass rules_folder and .yaml checks + if use_rule and os.path.isfile(use_rule): + return [use_rule] + rule_folder = conf['rules_folder'] + rule_files = [] + if 'scan_subdirectories' in conf and conf['scan_subdirectories']: + for root, folders, files in os.walk(rule_folder): + for filename in files: + if use_rule and use_rule != filename: + continue + if self.is_yaml(filename): + rule_files.append(os.path.join(root, filename)) + else: + for filename in os.listdir(rule_folder): + fullpath = os.path.join(rule_folder, filename) + if os.path.isfile(fullpath) and self.is_yaml(filename): + rule_files.append(fullpath) + return rule_files + + def get_hashes(self, conf, use_rule=None): + rule_files = self.get_names(conf, use_rule) + rule_mod_times = {} + for rule_file in rule_files: + rule_mod_times[rule_file] = self.get_rule_file_hash(rule_file) + return rule_mod_times + + def get_yaml(self, filename): + try: + return yaml_loader(filename) + except yaml.scanner.ScannerError as e: + raise EAException('Could not parse file %s: %s' % (filename, e)) + + def get_import_rule(self, rule): + """ + Allow for relative paths to the import rule. + :param dict rule: + :return: Path the import rule + :rtype: str + """ + if os.path.isabs(rule['import']): + return rule['import'] + else: + return os.path.join(os.path.dirname(rule['rule_file']), rule['import']) + + def get_rule_file_hash(self, rule_file): + rule_file_hash = '' + if os.path.exists(rule_file): + with open(rule_file, 'rb') as fh: + rule_file_hash = hashlib.sha1(fh.read()).digest() + for import_rule_file in self.import_rules.get(rule_file, []): + rule_file_hash += self.get_rule_file_hash(import_rule_file) + return rule_file_hash + + @staticmethod + def is_yaml(filename): + return filename.endswith('.yaml') or filename.endswith('.yml') diff --git a/elastalert/opsgenie.py b/elastalert/opsgenie.py index 45bf27dd1..bcdaf2d05 100644 --- a/elastalert/opsgenie.py +++ b/elastalert/opsgenie.py @@ -1,12 +1,14 @@ # -*- coding: utf-8 -*- import json import logging +import os.path import requests -from alerts import Alerter -from alerts import BasicMatchString -from util import EAException -from util import elastalert_logger -from util import lookup_es_key + +from .alerts import Alerter +from .alerts import BasicMatchString +from .util import EAException +from .util import elastalert_logger +from .util import lookup_es_key class OpsGenieAlerter(Alerter): @@ -15,11 +17,14 @@ class OpsGenieAlerter(Alerter): def __init__(self, *args): super(OpsGenieAlerter, self).__init__(*args) - self.account = self.rule.get('opsgenie_account') self.api_key = self.rule.get('opsgenie_key', 'key') + self.default_reciepients = self.rule.get('opsgenie_default_receipients', None) self.recipients = self.rule.get('opsgenie_recipients') + self.recipients_args = self.rule.get('opsgenie_recipients_args') + self.default_teams = self.rule.get('opsgenie_default_teams', None) self.teams = self.rule.get('opsgenie_teams') + self.teams_args = self.rule.get('opsgenie_teams_args') self.tags = self.rule.get('opsgenie_tags', []) + ['ElastAlert', self.rule['name']] self.to_addr = self.rule.get('opsgenie_addr', 'https://api.opsgenie.com/v2/alerts') self.custom_message = self.rule.get('opsgenie_message') @@ -28,6 +33,29 @@ def __init__(self, *args): self.alias = self.rule.get('opsgenie_alias') self.opsgenie_proxy = self.rule.get('opsgenie_proxy', None) self.priority = self.rule.get('opsgenie_priority') + self.opsgenie_details = self.rule.get('opsgenie_details', {}) + + def _parse_responders(self, responders, responder_args, matches, default_responders): + if responder_args: + formated_responders = list() + responders_values = dict((k, lookup_es_key(matches[0], v)) for k, v in responder_args.items()) + responders_values = dict((k, v) for k, v in responders_values.items() if v) + + for responder in responders: + responder = str(responder) + try: + formated_responders.append(responder.format(**responders_values)) + except KeyError as error: + logging.warn("OpsGenieAlerter: Cannot create responder for OpsGenie Alert. Key not foud: %s. " % (error)) + if not formated_responders: + logging.warn("OpsGenieAlerter: no responders can be formed. Trying the default responder ") + if not default_responders: + logging.warn("OpsGenieAlerter: default responder not set. Falling back") + formated_responders = responders + else: + formated_responders = default_responders + responders = formated_responders + return responders def _fill_responders(self, responders, type_): return [{'id': r, 'type': type_} for r in responders] @@ -35,7 +63,7 @@ def _fill_responders(self, responders, type_): def alert(self, matches): body = '' for match in matches: - body += unicode(BasicMatchString(self.rule, match)) + body += str(BasicMatchString(self.rule, match)) # Separate text of aggregated alerts with dashes if len(matches) > 1: body += '\n----------------------------------------\n' @@ -44,18 +72,23 @@ def alert(self, matches): self.message = self.create_title(matches) else: self.message = self.custom_message.format(**matches[0]) - + self.recipients = self._parse_responders(self.recipients, self.recipients_args, matches, self.default_reciepients) + self.teams = self._parse_responders(self.teams, self.teams_args, matches, self.default_teams) post = {} post['message'] = self.message if self.account: post['user'] = self.account if self.recipients: - post['responders'] = self._fill_responders(self.recipients, 'user') + post['responders'] = [{'username': r, 'type': 'user'} for r in self.recipients] if self.teams: - post['teams'] = self._fill_responders(self.teams, 'team') + post['teams'] = [{'name': r, 'type': 'team'} for r in self.teams] post['description'] = body post['source'] = 'ElastAlert' + + for i, tag in enumerate(self.tags): + self.tags[i] = tag.format(**matches[0]) post['tags'] = self.tags + if self.priority and self.priority not in ('P1', 'P2', 'P3', 'P4', 'P5'): logging.warn("Priority level does not appear to be specified correctly. \ Please make sure to set it to a value between P1 and P5") @@ -65,6 +98,10 @@ def alert(self, matches): if self.alias is not None: post['alias'] = self.alias.format(**matches[0]) + details = self.get_details(matches) + if details: + post['details'] = details + logging.debug(json.dumps(post)) headers = { @@ -105,7 +142,7 @@ def create_title(self, matches): return self.create_default_title(matches) def create_custom_title(self, matches): - opsgenie_subject = unicode(self.rule['opsgenie_subject']) + opsgenie_subject = str(self.rule['opsgenie_subject']) if self.opsgenie_subject_args: opsgenie_subject_values = [lookup_es_key(matches[0], arg) for arg in self.opsgenie_subject_args] @@ -129,5 +166,20 @@ def get_info(self): ret['account'] = self.account if self.teams: ret['teams'] = self.teams - return ret + + def get_details(self, matches): + details = {} + + for key, value in self.opsgenie_details.items(): + + if type(value) is dict: + if 'field' in value: + field_value = lookup_es_key(matches[0], value['field']) + if field_value is not None: + details[key] = str(field_value) + + elif type(value) is str: + details[key] = os.path.expandvars(value) + + return details diff --git a/elastalert/rule_from_kibana.py b/elastalert/rule_from_kibana.py index 7589d99d6..4a0634954 100644 --- a/elastalert/rule_from_kibana.py +++ b/elastalert/rule_from_kibana.py @@ -1,24 +1,32 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function - import json import yaml -from elasticsearch.client import Elasticsearch from elastalert.kibana import filters_from_dashboard +from elastalert.util import elasticsearch_client def main(): - es_host = raw_input("Elasticsearch host: ") - es_port = raw_input("Elasticsearch port: ") - db_name = raw_input("Dashboard name: ") - send_get_body_as = raw_input("Method for querying Elasticsearch[GET]: ") or 'GET' - es = Elasticsearch(host=es_host, port=es_port, send_get_body_as=send_get_body_as) + es_host = input("Elasticsearch host: ") + es_port = input("Elasticsearch port: ") + db_name = input("Dashboard name: ") + send_get_body_as = input("Method for querying Elasticsearch[GET]: ") or 'GET' + + es = elasticsearch_client({'es_host': es_host, 'es_port': es_port, 'send_get_body_as': send_get_body_as}) + + print("Elastic Version:" + es.es_version) + query = {'query': {'term': {'_id': db_name}}} - res = es.search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) + + if es.is_atleastsixsix(): + # TODO check support for kibana 7 + # TODO use doc_type='_doc' instead + res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_includes=['dashboard']) + else: + res = es.deprecated_search(index='kibana-int', doc_type='dashboard', body=query, _source_include=['dashboard']) + if not res['hits']['hits']: print("No dashboard %s found" % (db_name)) exit() diff --git a/elastalert/ruletypes.py b/elastalert/ruletypes.py index 1a53fd87d..2f1d2f82c 100644 --- a/elastalert/ruletypes.py +++ b/elastalert/ruletypes.py @@ -4,19 +4,20 @@ import sys from blist import sortedlist -from util import add_raw_postfix -from util import dt_to_ts -from util import EAException -from util import elastalert_logger -from util import elasticsearch_client -from util import format_index -from util import hashable -from util import lookup_es_key -from util import new_get_event_ts -from util import pretty_ts -from util import total_seconds -from util import ts_now -from util import ts_to_dt + +from .util import add_raw_postfix +from .util import dt_to_ts +from .util import EAException +from .util import elastalert_logger +from .util import elasticsearch_client +from .util import format_index +from .util import hashable +from .util import lookup_es_key +from .util import new_get_event_ts +from .util import pretty_ts +from .util import total_seconds +from .util import ts_now +from .util import ts_to_dt class RuleType(object): @@ -31,6 +32,8 @@ def __init__(self, rules, args=None): self.matches = [] self.rules = rules self.occurrences = {} + self.rules['category'] = self.rules.get('category', '') + self.rules['description'] = self.rules.get('description', '') self.rules['owner'] = self.rules.get('owner', '') self.rules['priority'] = self.rules.get('priority', '2') @@ -203,8 +206,8 @@ def add_match(self, match): if change: extra = {'old_value': change[0], 'new_value': change[1]} - elastalert_logger.debug("Description of the changed records " + str(dict(match.items() + extra.items()))) - super(ChangeRule, self).add_match(dict(match.items() + extra.items())) + elastalert_logger.debug("Description of the changed records " + str(dict(list(match.items()) + list(extra.items())))) + super(ChangeRule, self).add_match(dict(list(match.items()) + list(extra.items()))) class FrequencyRule(RuleType): @@ -222,14 +225,14 @@ def add_count_data(self, data): if len(data) > 1: raise EAException('add_count_data can only accept one count at a time') - (ts, count), = data.items() + (ts, count), = list(data.items()) event = ({self.ts_field: ts}, count) self.occurrences.setdefault('all', EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts)).append(event) self.check_for_match('all') def add_terms_data(self, terms): - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: event = ({self.ts_field: timestamp, self.rules['query_key']: bucket['key']}, bucket['doc_count']) @@ -272,10 +275,10 @@ def check_for_match(self, key, end=False): def garbage_collect(self, timestamp): """ Remove all occurrence data that is beyond the timeframe away """ stale_keys = [] - for key, window in self.occurrences.iteritems(): + for key, window in self.occurrences.items(): if timestamp - lookup_es_key(window.data[-1][0], self.ts_field) > self.rules['timeframe']: stale_keys.append(key) - map(self.occurrences.pop, stale_keys) + list(map(self.occurrences.pop, stale_keys)) def get_match_str(self, match): lt = self.rules.get('use_local_time') @@ -399,11 +402,11 @@ def add_count_data(self, data): """ Add count data to the rule. Data should be of the form {ts: count}. """ if len(data) > 1: raise EAException('add_count_data can only accept one count at a time') - for ts, count in data.iteritems(): + for ts, count in data.items(): self.handle_event({self.ts_field: ts}, count, 'all') def add_terms_data(self, terms): - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: count = bucket['doc_count'] event = {self.ts_field: timestamp, @@ -419,15 +422,14 @@ def add_data(self, data): if qk is None: qk = 'other' if self.field_value is not None: - if self.field_value in event: - count = lookup_es_key(event, self.field_value) - if count is not None: - try: - count = int(count) - except ValueError: - elastalert_logger.warn('{} is not a number: {}'.format(self.field_value, count)) - else: - self.handle_event(event, count, qk) + count = lookup_es_key(event, self.field_value) + if count is not None: + try: + count = int(count) + except ValueError: + elastalert_logger.warn('{} is not a number: {}'.format(self.field_value, count)) + else: + self.handle_event(event, count, qk) else: self.handle_event(event, 1, qk) @@ -435,7 +437,7 @@ def clear_windows(self, qk, event): # Reset the state and prevent alerts until windows filled again self.ref_windows[qk].clear() self.first_event.pop(qk) - self.skip_checks[qk] = event[self.ts_field] + self.rules['timeframe'] * 2 + self.skip_checks[qk] = lookup_es_key(event, self.ts_field) + self.rules['timeframe'] * 2 def handle_event(self, event, count, qk='all'): self.first_event.setdefault(qk, event) @@ -446,7 +448,7 @@ def handle_event(self, event, count, qk='all'): self.cur_windows[qk].append((event, count)) # Don't alert if ref window has not yet been filled for this key AND - if event[self.ts_field] - self.first_event[qk][self.ts_field] < self.rules['timeframe'] * 2: + if lookup_es_key(event, self.ts_field) - self.first_event[qk][self.ts_field] < self.rules['timeframe'] * 2: # ElastAlert has not been running long enough for any alerts OR if not self.ref_window_filled_once: return @@ -454,7 +456,7 @@ def handle_event(self, event, count, qk='all'): if not (self.rules.get('query_key') and self.rules.get('alert_on_new_data')): return # An alert for this qk has recently fired - if qk in self.skip_checks and event[self.ts_field] < self.skip_checks[qk]: + if qk in self.skip_checks and lookup_es_key(event, self.ts_field) < self.skip_checks[qk]: return else: self.ref_window_filled_once = True @@ -488,7 +490,7 @@ def add_match(self, match, qk): extra_info = {'spike_count': spike_count, 'reference_count': reference_count} - match = dict(match.items() + extra_info.items()) + match = dict(list(match.items()) + list(extra_info.items())) super(SpikeRule, self).add_match(match) @@ -534,7 +536,7 @@ def get_match_str(self, match): def garbage_collect(self, ts): # Windows are sized according to their newest event # This is a placeholder to accurately size windows in the absence of events - for qk in self.cur_windows.keys(): + for qk in list(self.cur_windows.keys()): # If we havn't seen this key in a long time, forget it if qk != 'all' and self.ref_windows[qk].count() == 0 and self.cur_windows[qk].count() == 0: self.cur_windows.pop(qk) @@ -607,7 +609,7 @@ def garbage_collect(self, ts): # We add an event with a count of zero to the EventWindow for each key. This will cause the EventWindow # to remove events that occurred more than one `timeframe` ago, and call onRemoved on them. default = ['all'] if 'query_key' not in self.rules else [] - for key in self.occurrences.keys() or default: + for key in list(self.occurrences.keys()) or default: self.occurrences.setdefault( key, EventWindow(self.rules['timeframe'], getTimestamp=self.get_ts) @@ -639,7 +641,7 @@ def __init__(self, rule, args=None): (len(self.fields) != 1 or (len(self.fields) == 1 and type(self.fields[0]) == list)): raise EAException("use_terms_query can only be used with a single non-composite field") if self.rules.get('use_terms_query'): - if self.rules.get('query_key') != self.fields: + if [self.rules['query_key']] != self.fields: raise EAException('If use_terms_query is specified, you cannot specify different query_key and fields') if not self.rules.get('query_key').endswith('.keyword') and not self.rules.get('query_key').endswith('.raw'): if self.rules.get('use_keyword_postfix', True): @@ -649,7 +651,7 @@ def __init__(self, rule, args=None): self.get_all_terms(args) except Exception as e: # Refuse to start if we cannot get existing terms - raise EAException('Error searching for existing terms: %s' % (repr(e))), None, sys.exc_info()[2] + raise EAException('Error searching for existing terms: %s' % (repr(e))).with_traceback(sys.exc_info()[2]) def get_all_terms(self, args): """ Performs a terms aggregation for each field to get every existing term. """ @@ -730,7 +732,7 @@ def get_all_terms(self, args): time_filter[self.rules['timestamp_field']] = {'lt': self.rules['dt_to_ts'](tmp_end), 'gte': self.rules['dt_to_ts'](tmp_start)} - for key, values in self.seen_values.iteritems(): + for key, values in self.seen_values.items(): if not values: if type(key) == tuple: # If we don't have any results, it could either be because of the absence of any baseline data @@ -878,7 +880,7 @@ def add_data(self, data): def add_terms_data(self, terms): # With terms query, len(self.fields) is always 1 and the 0'th entry is always a string field = self.fields[0] - for timestamp, buckets in terms.iteritems(): + for timestamp, buckets in terms.items(): for bucket in buckets: if bucket['doc_count']: if bucket['key'] not in self.seen_values[field]: @@ -916,22 +918,23 @@ def add_data(self, data): # If no query_key, we use the key 'all' for all events key = 'all' self.cardinality_cache.setdefault(key, {}) - self.first_event.setdefault(key, event[self.ts_field]) + self.first_event.setdefault(key, lookup_es_key(event, self.ts_field)) value = hashable(lookup_es_key(event, self.cardinality_field)) if value is not None: # Store this timestamp as most recent occurence of the term - self.cardinality_cache[key][value] = event[self.ts_field] + self.cardinality_cache[key][value] = lookup_es_key(event, self.ts_field) self.check_for_match(key, event) def check_for_match(self, key, event, gc=True): # Check to see if we are past max/min_cardinality for a given key - timeframe_elapsed = event[self.ts_field] - self.first_event.get(key, event[self.ts_field]) > self.timeframe + time_elapsed = lookup_es_key(event, self.ts_field) - self.first_event.get(key, lookup_es_key(event, self.ts_field)) + timeframe_elapsed = time_elapsed > self.timeframe if (len(self.cardinality_cache[key]) > self.rules.get('max_cardinality', float('inf')) or (len(self.cardinality_cache[key]) < self.rules.get('min_cardinality', float('-inf')) and timeframe_elapsed)): # If there might be a match, run garbage collect first, as outdated terms are only removed in GC # Only run it if there might be a match so it doesn't impact performance if gc: - self.garbage_collect(event[self.ts_field]) + self.garbage_collect(lookup_es_key(event, self.ts_field)) self.check_for_match(key, event, False) else: self.first_event.pop(key, None) @@ -939,8 +942,8 @@ def check_for_match(self, key, event, gc=True): def garbage_collect(self, timestamp): """ Remove all occurrence data that is beyond the timeframe away """ - for qk, terms in self.cardinality_cache.items(): - for term, last_occurence in terms.items(): + for qk, terms in list(self.cardinality_cache.items()): + for term, last_occurence in list(terms.items()): if timestamp - last_occurence > self.rules['timeframe']: self.cardinality_cache[qk].pop(term) @@ -953,8 +956,8 @@ def garbage_collect(self, timestamp): def get_match_str(self, match): lt = self.rules.get('use_local_time') - starttime = pretty_ts(dt_to_ts(ts_to_dt(match[self.ts_field]) - self.rules['timeframe']), lt) - endtime = pretty_ts(match[self.ts_field], lt) + starttime = pretty_ts(dt_to_ts(ts_to_dt(lookup_es_key(match, self.ts_field)) - self.rules['timeframe']), lt) + endtime = pretty_ts(lookup_es_key(match, self.ts_field), lt) if 'max_cardinality' in self.rules: message = ('A maximum of %d unique %s(s) occurred since last alert or between %s and %s\n\n' % (self.rules['max_cardinality'], self.rules['cardinality_field'], @@ -995,7 +998,7 @@ def generate_aggregation_query(self): raise NotImplementedError() def add_aggregation_data(self, payload): - for timestamp, payload_data in payload.iteritems(): + for timestamp, payload_data in payload.items(): if 'interval_aggs' in payload_data: self.unwrap_interval_buckets(timestamp, None, payload_data['interval_aggs']['buckets']) elif 'bucket_aggs' in payload_data: @@ -1021,7 +1024,7 @@ def check_matches(self, timestamp, query_key, aggregation_data): class MetricAggregationRule(BaseAggregationRule): """ A rule that matches when there is a low number of events given a timeframe. """ - required_options = frozenset(['metric_agg_key', 'metric_agg_type', 'doc_type']) + required_options = frozenset(['metric_agg_key', 'metric_agg_type']) allowed_aggregations = frozenset(['min', 'max', 'avg', 'sum', 'cardinality', 'value_count']) def __init__(self, *args): @@ -1030,7 +1033,7 @@ def __init__(self, *args): if 'max_threshold' not in self.rules and 'min_threshold' not in self.rules: raise EAException("MetricAggregationRule must have at least one of either max_threshold or min_threshold") - self.metric_key = self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type'] + self.metric_key = 'metric_' + self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type'] if not self.rules['metric_agg_type'] in self.allowed_aggregations: raise EAException("metric_agg_type must be one of %s" % (str(self.allowed_aggregations))) @@ -1085,7 +1088,7 @@ def check_matches_recursive(self, timestamp, query_key, aggregation_data, compou # add compound key to payload to allow alerts to trigger for every unique occurence compound_value = [match_data[key] for key in self.rules['compound_query_key']] - match_data[self.rules['query_key']] = ",".join(compound_value) + match_data[self.rules['query_key']] = ",".join([str(value) for value in compound_value]) self.add_match(match_data) @@ -1099,6 +1102,88 @@ def crossed_thresholds(self, metric_value): return False +class SpikeMetricAggregationRule(BaseAggregationRule, SpikeRule): + """ A rule that matches when there is a spike in an aggregated event compared to its reference point """ + required_options = frozenset(['metric_agg_key', 'metric_agg_type', 'spike_height', 'spike_type']) + allowed_aggregations = frozenset(['min', 'max', 'avg', 'sum', 'cardinality', 'value_count']) + + def __init__(self, *args): + # We inherit everything from BaseAggregation and Spike, overwrite only what we need in functions below + super(SpikeMetricAggregationRule, self).__init__(*args) + + # MetricAgg alert things + self.metric_key = 'metric_' + self.rules['metric_agg_key'] + '_' + self.rules['metric_agg_type'] + if not self.rules['metric_agg_type'] in self.allowed_aggregations: + raise EAException("metric_agg_type must be one of %s" % (str(self.allowed_aggregations))) + + # Disabling bucket intervals (doesn't make sense in context of spike to split up your time period) + if self.rules.get('bucket_interval'): + raise EAException("bucket intervals are not supported for spike aggregation alerts") + + self.rules['aggregation_query_element'] = self.generate_aggregation_query() + + def generate_aggregation_query(self): + """Lifted from MetricAggregationRule, added support for scripted fields""" + if self.rules.get('metric_agg_script'): + return {self.metric_key: {self.rules['metric_agg_type']: self.rules['metric_agg_script']}} + return {self.metric_key: {self.rules['metric_agg_type']: {'field': self.rules['metric_agg_key']}}} + + def add_aggregation_data(self, payload): + """ + BaseAggregationRule.add_aggregation_data unpacks our results and runs checks directly against hardcoded cutoffs. + We instead want to use all of our SpikeRule.handle_event inherited logic (current/reference) from + the aggregation's "value" key to determine spikes from aggregations + """ + for timestamp, payload_data in payload.items(): + if 'bucket_aggs' in payload_data: + self.unwrap_term_buckets(timestamp, payload_data['bucket_aggs']) + else: + # no time / term split, just focus on the agg + event = {self.ts_field: timestamp} + agg_value = payload_data[self.metric_key]['value'] + self.handle_event(event, agg_value, 'all') + return + + def unwrap_term_buckets(self, timestamp, term_buckets, qk=[]): + """ + create separate spike event trackers for each term, + handle compound query keys + """ + for term_data in term_buckets['buckets']: + qk.append(term_data['key']) + + # handle compound query keys (nested aggregations) + if term_data.get('bucket_aggs'): + self.unwrap_term_buckets(timestamp, term_data['bucket_aggs'], qk) + # reset the query key to consider the proper depth for N > 2 + del qk[-1] + continue + + qk_str = ','.join(qk) + agg_value = term_data[self.metric_key]['value'] + event = {self.ts_field: timestamp, + self.rules['query_key']: qk_str} + # pass to SpikeRule's tracker + self.handle_event(event, agg_value, qk_str) + + # handle unpack of lowest level + del qk[-1] + return + + def get_match_str(self, match): + """ + Overwrite SpikeRule's message to relate to the aggregation type & field instead of count + """ + message = 'An abnormal {0} of {1} ({2}) occurred around {3}.\n'.format( + self.rules['metric_agg_type'], self.rules['metric_agg_key'], round(match['spike_count'], 2), + pretty_ts(match[self.rules['timestamp_field']], self.rules.get('use_local_time')) + ) + message += 'Preceding that time, there was a {0} of {1} of ({2}) within {3}\n\n'.format( + self.rules['metric_agg_type'], self.rules['metric_agg_key'], + round(match['reference_count'], 2), self.rules['timeframe']) + return message + + class PercentageMatchRule(BaseAggregationRule): required_options = frozenset(['match_bucket_filter']) @@ -1108,6 +1193,7 @@ def __init__(self, *args): if 'max_percentage' not in self.rules and 'min_percentage' not in self.rules: raise EAException("PercentageMatchRule must have at least one of either min_percentage or max_percentage") + self.min_denominator = self.rules.get('min_denominator', 0) self.match_bucket_filter = self.rules['match_bucket_filter'] self.rules['aggregation_query_element'] = self.generate_aggregation_query() @@ -1145,7 +1231,7 @@ def check_matches(self, timestamp, query_key, aggregation_data): return else: total_count = other_bucket_count + match_bucket_count - if total_count == 0: + if total_count == 0 or total_count < self.min_denominator: return else: match_percentage = (match_bucket_count * 1.0) / (total_count * 1.0) * 100 diff --git a/elastalert/schema.yaml b/elastalert/schema.yaml index 4afad9f1f..1241315dc 100644 --- a/elastalert/schema.yaml +++ b/elastalert/schema.yaml @@ -1,4 +1,4 @@ -$schema: http://json-schema.org/draft-04/schema# +$schema: http://json-schema.org/draft-07/schema# definitions: # Either a single string OR an array of strings @@ -11,6 +11,17 @@ definitions: type: [string, array] items: {type: [string, array]} + timedelta: &timedelta + type: object + additionalProperties: false + properties: + days: {type: number} + weeks: {type: number} + hours: {type: number} + minutes: {type: number} + seconds: {type: number} + milliseconds: {type: number} + timeFrame: &timeframe type: object additionalProperties: false @@ -25,6 +36,15 @@ definitions: filter: &filter {} + mattermostField: &mattermostField + type: object + additionalProperties: false + properties: + title: {type: string} + value: {type: string} + args: *arrayOfString + short: {type: boolean} + required: [type, index, alert] type: object @@ -84,6 +104,23 @@ oneOf: threshold_ref: {type: integer} threshold_cur: {type: integer} + - title: Spike Aggregation + required: [spike_height, spike_type, timeframe] + properties: + type: {enum: [spike_aggregation]} + spike_height: {type: number} + spike_type: {enum: ["up", "down", "both"]} + metric_agg_type: {enum: ["min", "max", "avg", "sum", "cardinality", "value_count"]} + timeframe: *timeframe + use_count_query: {type: boolean} + doc_type: {type: string} + use_terms_query: {type: boolean} + terms_size: {type: integer} + alert_on_new_data: {type: boolean} + threshold_ref: {type: number} + threshold_cur: {type: number} + min_doc_count: {type: integer} + - title: Flatline required: [threshold, timeframe] properties: @@ -94,7 +131,7 @@ oneOf: doc_type: {type: string} - title: New Term - required: [fields] + required: [] properties: type: {enum: [new_term]} fields: *arrayOfStringsOrOtherArray @@ -152,6 +189,7 @@ properties: buffer_time: *timeframe query_delay: *timeframe max_query_size: {type: integer} + max_scrolling: {type: integer} owner: {type: string} priority: {type: integer} @@ -176,6 +214,15 @@ properties: replace_dots_in_field_names: {type: boolean} scan_entire_timeframe: {type: boolean} + ### Kibana Discover App Link + generate_kibana_discover_url: {type: boolean} + kibana_discover_app_url: {type: string, format: uri} + kibana_discover_version: {type: string, enum: ['7.3', '7.2', '7.1', '7.0', '6.8', '6.7', '6.6', '6.5', '6.4', '6.3', '6.2', '6.1', '6.0', '5.6']} + kibana_discover_index_pattern_id: {type: string, minLength: 1} + kibana_discover_columns: {type: array, items: {type: string, minLength: 1}, minItems: 1} + kibana_discover_from_timedelta: *timedelta + kibana_discover_to_timedelta: *timedelta + # Alert Content alert_text: {type: string} # Python format string alert_text_args: {type: array, items: {type: string}} @@ -238,12 +285,45 @@ properties: slack_parse_override: {enum: [none, full]} slack_text_string: {type: string} slack_ignore_ssl_errors: {type: boolean} + slack_ca_certs: {type: string} + slack_attach_kibana_discover_url: {type: boolean} + slack_kibana_discover_color: {type: string} + slack_kibana_discover_title: {type: string} + + ### Mattermost + mattermost_webhook_url: *arrayOfString + mattermost_proxy: {type: string} + mattermost_ignore_ssl_errors: {type: boolean} + mattermost_username_override: {type: string} + mattermost_icon_url_override: {type: string} + mattermost_channel_override: {type: string} + mattermost_msg_color: {enum: [good, warning, danger]} + mattermost_msg_pretext: {type: string} + mattermost_msg_fields: *mattermostField + + ## Opsgenie + opsgenie_details: + type: object + minProperties: 1 + patternProperties: + "^.+$": + oneOf: + - type: string + - type: object + additionalProperties: false + required: [field] + properties: + field: {type: string, minLength: 1} ### PagerDuty pagerduty_service_key: {type: string} pagerduty_client_name: {type: string} pagerduty_event_type: {enum: [none, trigger, resolve, acknowledge]} +### PagerTree + pagertree_integration_url: {type: string} + + ### Exotel exotel_account_sid: {type: string} exotel_auth_token: {type: string} @@ -298,3 +378,12 @@ properties: ### Simple simple_webhook_url: *arrayOfString simple_proxy: {type: string} + + ### LineNotify + linenotify_access_token: {type: string} + + ### Zabbix + zbx_sender_host: {type: string} + zbx_sender_port: {type: integer} + zbx_host: {type: string} + zbx_item: {type: string} diff --git a/elastalert/test_rule.py b/elastalert/test_rule.py index 94215bdac..06100aa0f 100644 --- a/elastalert/test_rule.py +++ b/elastalert/test_rule.py @@ -1,27 +1,20 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -from __future__ import absolute_import -from __future__ import print_function - +import argparse import copy import datetime import json import logging -import os import random import re import string import sys -import argparse import mock -import yaml -import elastalert.config -from elastalert.config import load_modules -from elastalert.config import load_options -from elastalert.config import load_rule_yaml +from elastalert.config import load_conf from elastalert.elastalert import ElastAlerter +from elastalert.util import EAException from elastalert.util import elasticsearch_client from elastalert.util import lookup_es_key from elastalert.util import ts_now @@ -30,6 +23,14 @@ logging.getLogger().setLevel(logging.INFO) logging.getLogger('elasticsearch').setLevel(logging.WARNING) +""" +Error Codes: + 1: Error connecting to ElasticSearch + 2: Error querying ElasticSearch + 3: Invalid Rule + 4: Missing/invalid timestamp +""" + def print_terms(terms, parent): """ Prints a list of flattened dictionary keys """ @@ -43,6 +44,7 @@ def print_terms(terms, parent): class MockElastAlerter(object): def __init__(self): self.data = [] + self.formatted_output = {} def test_file(self, conf, args): """ Loads a rule config file, performs a query over the last day (args.days), lists available keys @@ -55,13 +57,17 @@ def test_file(self, conf, args): try: ElastAlerter.modify_rule_for_ES5(conf) + except EAException as ea: + print('Invalid filter provided:', str(ea), file=sys.stderr) + if args.stop_error: + exit(3) + return None except Exception as e: print("Error connecting to ElasticSearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: exit(1) return None - start_time = ts_now() - datetime.timedelta(days=args.days) end_time = ts_now() ts = conf.get('timestamp_field', '@timestamp') @@ -70,6 +76,7 @@ def test_file(self, conf, args): starttime=start_time, endtime=end_time, timestamp_field=ts, + to_ts_func=conf['dt_to_ts'], five=conf['five'] ) index = ElastAlerter.get_index(conf, start_time, end_time) @@ -81,10 +88,11 @@ def test_file(self, conf, args): print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(3) return None num_hits = len(res['hits']['hits']) if not num_hits: + print("Didn't get any results.") return [] terms = res['hits']['hits'][0]['_source'] @@ -96,6 +104,7 @@ def test_file(self, conf, args): starttime=start_time, endtime=end_time, timestamp_field=ts, + to_ts_func=conf['dt_to_ts'], sort=False, five=conf['five'] ) @@ -105,13 +114,20 @@ def test_file(self, conf, args): print("Error querying Elasticsearch:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(2) return None num_hits = res['count'] - print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else '')) - print("\nAvailable terms in first hit:") - print_terms(terms, '') + + if args.formatted_output: + self.formatted_output['hits'] = num_hits + self.formatted_output['days'] = args.days + self.formatted_output['terms'] = list(terms.keys()) + self.formatted_output['result'] = terms + else: + print("Got %s hits from the last %s day%s" % (num_hits, args.days, 's' if args.days > 1 else '')) + print("\nAvailable terms in first hit:") + print_terms(terms, '') # Check for missing keys pk = conf.get('primary_key') @@ -131,20 +147,23 @@ def test_file(self, conf, args): # If the index starts with 'logstash', fields with .raw will be available but won't in _source if term not in terms and not (term.endswith('.raw') and term[:-4] in terms and index.startswith('logstash')): print("top_count_key %s may be missing" % (term), file=sys.stderr) - print('') # Newline + if not args.formatted_output: + print('') # Newline # Download up to max_query_size (defaults to 10,000) documents to save - if args.save and not args.count: + if (args.save or args.formatted_output) and not args.count: try: res = es_client.search(index, size=args.max_query_size, body=query, ignore_unavailable=True) except Exception as e: print("Error running your filter:", file=sys.stderr) print(repr(e)[:2048], file=sys.stderr) if args.stop_error: - exit(1) + exit(2) return None num_hits = len(res['hits']['hits']) - print("Downloaded %s documents to save" % (num_hits)) + + if args.save: + print("Downloaded %s documents to save" % (num_hits)) return res['hits']['hits'] def mock_count(self, rule, start, end, index): @@ -169,7 +188,7 @@ def mock_hits(self, rule, start, end, index, scroll=False): if field != '_id': if not any([re.match(incl.replace('*', '.*'), field) for incl in rule['include']]): fields_to_remove.append(field) - map(doc.pop, fields_to_remove) + list(map(doc.pop, fields_to_remove)) # Separate _source and _id, convert timestamps resp = [{'_source': doc, '_id': doc['_id']} for doc in docs] @@ -189,7 +208,7 @@ def mock_terms(self, rule, start, end, index, key, qk=None, size=None): if qk is None or doc[rule['query_key']] == qk: buckets.setdefault(doc[key], 0) buckets[doc[key]] += 1 - counts = buckets.items() + counts = list(buckets.items()) counts.sort(key=lambda x: x[1], reverse=True) if size: counts = counts[:size] @@ -211,8 +230,7 @@ def run_elastalert(self, rule, conf, args): # It is needed to prevent unnecessary initialization of unused alerters load_modules_args = argparse.Namespace() load_modules_args.debug = not args.alert - load_modules(rule, load_modules_args) - conf['rules'] = [rule] + conf['rules_loader'].load_modules(rule, load_modules_args) # If using mock data, make sure it's sorted and find appropriate time range timestamp_field = rule.get('timestamp_field', '@timestamp') @@ -227,14 +245,14 @@ def run_elastalert(self, rule, conf, args): except KeyError as e: print("All documents must have a timestamp and _id: %s" % (e), file=sys.stderr) if args.stop_error: - exit(1) + exit(4) return None # Create mock _id for documents if it's missing used_ids = [] def get_id(): - _id = ''.join([random.choice(string.letters) for i in range(16)]) + _id = ''.join([random.choice(string.ascii_letters) for i in range(16)]) if _id in used_ids: return get_id() used_ids.append(_id) @@ -251,7 +269,7 @@ def get_id(): endtime = ts_to_dt(args.end) except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (args.end)) - exit(1) + exit(4) else: endtime = ts_now() if args.start: @@ -259,9 +277,18 @@ def get_id(): starttime = ts_to_dt(args.start) except (TypeError, ValueError): self.handle_error("%s is not a valid ISO8601 timestamp (YYYY-MM-DDTHH:MM:SS+XX:00)" % (args.start)) - exit(1) + exit(4) else: - starttime = endtime - datetime.timedelta(days=args.days) + # if days given as command line argument + if args.days > 0: + starttime = endtime - datetime.timedelta(days=args.days) + else: + # if timeframe is given in rule + if 'timeframe' in rule: + starttime = endtime - datetime.timedelta(seconds=rule['timeframe'].total_seconds() * 1.01) + # default is 1 days / 24 hours + else: + starttime = endtime - datetime.timedelta(days=1) # Set run_every to cover the entire time range unless count query, terms query or agg query used # This is to prevent query segmenting which unnecessarily slows down tests @@ -269,13 +296,15 @@ def get_id(): conf['run_every'] = endtime - starttime # Instantiate ElastAlert to use mock config and special rule - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: - load_conf.return_value = conf - if args.alert: - client = ElastAlerter(['--verbose']) - else: - client = ElastAlerter(['--debug']) + with mock.patch.object(conf['rules_loader'], 'get_hashes'): + with mock.patch.object(conf['rules_loader'], 'load') as load_rules: + load_rules.return_value = [rule] + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + load_conf.return_value = conf + if args.alert: + client = ElastAlerter(['--verbose']) + else: + client = ElastAlerter(['--debug']) # Replace get_hits_* functions to use mock data if args.json: @@ -289,58 +318,23 @@ def get_id(): client.run_rule(rule, endtime, starttime) if mock_writeback.call_count: - print("\nWould have written the following documents to writeback index (default is elastalert_status):\n") + + if args.formatted_output: + self.formatted_output['writeback'] = {} + else: + print("\nWould have written the following documents to writeback index (default is elastalert_status):\n") + errors = False for call in mock_writeback.call_args_list: - print("%s - %s\n" % (call[0][0], call[0][1])) + if args.formatted_output: + self.formatted_output['writeback'][call[0][0]] = json.loads(json.dumps(call[0][1], default=str)) + else: + print("%s - %s\n" % (call[0][0], call[0][1])) + if call[0][0] == 'elastalert_error': errors = True if errors and args.stop_error: - exit(1) - - def load_conf(self, rules, args): - """ Loads a default conf dictionary (from global config file, if provided, or hard-coded mocked data), - for initializing rules. Also initializes rules. - - :return: the default rule configuration, a dictionary """ - if args.config is not None: - with open(args.config) as fh: - conf = yaml.load(fh) - else: - if os.path.isfile('config.yaml'): - with open('config.yaml') as fh: - conf = yaml.load(fh) - else: - conf = {} - - # Need to convert these parameters to datetime objects - for key in ['buffer_time', 'run_every', 'alert_time_limit', 'old_query_limit']: - if key in conf: - conf[key] = datetime.timedelta(**conf[key]) - - # Mock configuration. This specifies the base values for attributes, unless supplied otherwise. - conf_default = { - 'rules_folder': 'rules', - 'es_host': 'localhost', - 'es_port': 14900, - 'writeback_index': 'wb', - 'max_query_size': 10000, - 'alert_time_limit': datetime.timedelta(hours=24), - 'old_query_limit': datetime.timedelta(weeks=1), - 'run_every': datetime.timedelta(minutes=5), - 'disable_rules_on_error': False, - 'buffer_time': datetime.timedelta(minutes=45), - 'scroll_keepalive': '30s' - } - - for key in conf_default: - if key not in conf: - conf[key] = conf_default[key] - elastalert.config.base_config = copy.deepcopy(conf) - load_options(rules, conf, args.file) - print("Successfully loaded %s\n" % (rules['name'])) - - return conf + exit(2) def run_rule_test(self): """ @@ -349,11 +343,12 @@ def run_rule_test(self): parser = argparse.ArgumentParser(description='Validate a rule configuration') parser.add_argument('file', metavar='rule', type=str, help='rule configuration filename') parser.add_argument('--schema-only', action='store_true', help='Show only schema errors; do not run query') - parser.add_argument('--days', type=int, default=1, action='store', help='Query the previous N days with this rule') + parser.add_argument('--days', type=int, default=0, action='store', help='Query the previous N days with this rule') parser.add_argument('--start', dest='start', help='YYYY-MM-DDTHH:MM:SS Start querying from this timestamp.') parser.add_argument('--end', dest='end', help='YYYY-MM-DDTHH:MM:SS Query to this timestamp. (Default: present) ' 'Use "NOW" to start from current time. (Default: present)') parser.add_argument('--stop-error', action='store_true', help='Stop the entire test right after the first error') + parser.add_argument('--formatted-output', action='store_true', help='Output results in formatted JSON') parser.add_argument( '--data', type=str, @@ -390,20 +385,46 @@ def run_rule_test(self): parser.add_argument('--config', action='store', dest='config', help='Global config file.') args = parser.parse_args() - rule_yaml = load_rule_yaml(args.file) + defaults = { + 'rules_folder': 'rules', + 'es_host': 'localhost', + 'es_port': 14900, + 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', + 'max_query_size': 10000, + 'alert_time_limit': {'hours': 24}, + 'old_query_limit': {'weeks': 1}, + 'run_every': {'minutes': 5}, + 'disable_rules_on_error': False, + 'buffer_time': {'minutes': 45}, + 'scroll_keepalive': '30s' + } + overwrites = { + 'rules_loader': 'file', + } - conf = self.load_conf(rule_yaml, args) + # Set arguments that ElastAlerter needs + args.verbose = args.alert + args.debug = not args.alert + args.es_debug = False + args.es_debug_trace = False + + conf = load_conf(args, defaults, overwrites) + rule_yaml = conf['rules_loader'].load_yaml(args.file) + conf['rules_loader'].load_options(rule_yaml, conf, args.file) if args.json: with open(args.json, 'r') as data_file: self.data = json.loads(data_file.read()) else: hits = self.test_file(copy.deepcopy(rule_yaml), args) + if hits and args.formatted_output: + self.formatted_output['results'] = json.loads(json.dumps(hits)) if hits and args.save: with open(args.save, 'wb') as data_file: # Add _id to _source for dump [doc['_source'].update({'_id': doc['_id']}) for doc in hits] - data_file.write(json.dumps([doc['_source'] for doc in hits], indent=' ')) + data_file.write(json.dumps([doc['_source'] for doc in hits], indent=4)) if args.use_downloaded: if hits: args.json = args.save @@ -415,6 +436,9 @@ def run_rule_test(self): if not args.schema_only and not args.count: self.run_elastalert(rule_yaml, conf, args) + if args.formatted_output: + print(json.dumps(self.formatted_output)) + def main(): test_instance = MockElastAlerter() diff --git a/elastalert/util.py b/elastalert/util.py index 04e87baff..bbb0600ff 100644 --- a/elastalert/util.py +++ b/elastalert/util.py @@ -3,18 +3,34 @@ import datetime import logging import os +import re +import sys import dateutil.parser -import dateutil.tz -from auth import Auth -from elasticsearch import RequestsHttpConnection -from elasticsearch.client import Elasticsearch +import pytz from six import string_types +from . import ElasticSearchClient +from .auth import Auth + logging.basicConfig() elastalert_logger = logging.getLogger('elastalert') +def get_module(module_name): + """ Loads a module and returns a specific object. + module_name should 'module.file.object'. + Returns object or raises EAException on error. """ + sys.path.append(os.getcwd()) + try: + module_path, module_class = module_name.rsplit('.', 1) + base_module = __import__(module_path, globals(), locals(), [module_class]) + module = getattr(base_module, module_class) + except (ImportError, AttributeError, ValueError) as e: + raise EAException("Could not import module %s: %s" % (module_name, e)).with_traceback(sys.exc_info()[2]) + return module + + def new_get_event_ts(ts_field): """ Constructs a lambda that may be called to extract the timestamp field from a given event. @@ -60,27 +76,45 @@ def _find_es_dict_by_key(lookup_dict, term): # For example: # {'foo.bar': {'bar': 'ray'}} to look up foo.bar will return {'bar': 'ray'}, not 'ray' dict_cursor = lookup_dict - subkeys = term.split('.') - subkey = '' - while len(subkeys) > 0: - if not dict_cursor: - return {}, None + while term: + split_results = re.split(r'\[(\d)\]', term, maxsplit=1) + if len(split_results) == 3: + sub_term, index, term = split_results + index = int(index) + else: + sub_term, index, term = split_results + [None, ''] - subkey += subkeys.pop(0) + subkeys = sub_term.split('.') - if subkey in dict_cursor: - if len(subkeys) == 0: - break + subkey = '' + while len(subkeys) > 0: + if not dict_cursor: + return {}, None + + subkey += subkeys.pop(0) + + if subkey in dict_cursor: + if len(subkeys) == 0: + break + dict_cursor = dict_cursor[subkey] + subkey = '' + elif len(subkeys) == 0: + # If there are no keys left to match, return None values + dict_cursor = None + subkey = None + else: + subkey += '.' + + if index is not None and subkey: dict_cursor = dict_cursor[subkey] - subkey = '' - elif len(subkeys) == 0: - # If there are no keys left to match, return None values - dict_cursor = None - subkey = None - else: - subkey += '.' + if type(dict_cursor) == list and len(dict_cursor) > index: + subkey = index + if term: + dict_cursor = dict_cursor[subkey] + else: + return {}, None return dict_cursor, subkey @@ -112,7 +146,7 @@ def ts_to_dt(timestamp): dt = dateutil.parser.parse(timestamp) # Implicitly convert local timestamps to UTC if dt.tzinfo is None: - dt = dt.replace(tzinfo=dateutil.tz.tzutc()) + dt = dt.replace(tzinfo=pytz.utc) return dt @@ -281,7 +315,7 @@ def replace_dots_in_field_names(document): def elasticsearch_client(conf): - """ returns an Elasticsearch instance configured using an es_conn_config """ + """ returns an :class:`ElasticSearchClient` instance configured using an es_conn_config """ es_conn_conf = build_es_conn_config(conf) auth = Auth() es_conn_conf['http_auth'] = auth(host=es_conn_conf['es_host'], @@ -290,18 +324,7 @@ def elasticsearch_client(conf): aws_region=es_conn_conf['aws_region'], profile_name=es_conn_conf['profile']) - return Elasticsearch(host=es_conn_conf['es_host'], - port=es_conn_conf['es_port'], - url_prefix=es_conn_conf['es_url_prefix'], - use_ssl=es_conn_conf['use_ssl'], - verify_certs=es_conn_conf['verify_certs'], - ca_certs=es_conn_conf['ca_certs'], - connection_class=RequestsHttpConnection, - http_auth=es_conn_conf['http_auth'], - timeout=es_conn_conf['es_conn_timeout'], - send_get_body_as=es_conn_conf['send_get_body_as'], - client_cert=es_conn_conf['client_cert'], - client_key=es_conn_conf['client_key']) + return ElasticSearchClient(es_conn_conf) def build_es_conn_config(conf): @@ -326,9 +349,12 @@ def build_es_conn_config(conf): parsed_conf['es_conn_timeout'] = conf.get('es_conn_timeout', 20) parsed_conf['send_get_body_as'] = conf.get('es_send_get_body_as', 'GET') - if 'es_username' in conf: - parsed_conf['es_username'] = os.environ.get('ES_USERNAME', conf['es_username']) - parsed_conf['es_password'] = os.environ.get('ES_PASSWORD', conf['es_password']) + if os.environ.get('ES_USERNAME'): + parsed_conf['es_username'] = os.environ.get('ES_USERNAME') + parsed_conf['es_password'] = os.environ.get('ES_PASSWORD') + elif 'es_username' in conf: + parsed_conf['es_username'] = conf['es_username'] + parsed_conf['es_password'] = conf['es_password'] if 'aws_region' in conf: parsed_conf['aws_region'] = conf['aws_region'] @@ -362,6 +388,15 @@ def build_es_conn_config(conf): return parsed_conf +def pytzfy(dt): + # apscheduler requires pytz timezone objects + # This function will replace a dateutil.tz one with a pytz one + if dt.tzinfo is not None: + new_tz = pytz.timezone(dt.tzinfo.tzname('Y is this even required??')) + return dt.replace(tzinfo=new_tz) + return dt + + def parse_duration(value): """Convert ``unit=num`` spec into a ``timedelta`` object.""" unit, num = value.split('=') @@ -376,7 +411,7 @@ def parse_deadline(value): def flatten_dict(dct, delim='.', prefix=''): ret = {} - for key, val in dct.items(): + for key, val in list(dct.items()): if type(val) == dict: ret.update(flatten_dict(val, prefix=prefix + key + delim)) else: @@ -407,8 +442,21 @@ def resolve_string(string, match, missing_text=''): string = string.format(**dd_match) break except KeyError as e: - if '{%s}' % e.message not in string: + if '{%s}' % str(e).strip("'") not in string: break - string = string.replace('{%s}' % e.message, '{_missing_value}') + string = string.replace('{%s}' % str(e).strip("'"), '{_missing_value}') return string + + +def should_scrolling_continue(rule_conf): + """ + Tells about a rule config if it can scroll still or should stop the scrolling. + + :param: rule_conf as dict + :rtype: bool + """ + max_scrolling = rule_conf.get('max_scrolling_count') + stop_the_scroll = 0 < max_scrolling <= rule_conf.get('scrolling_cycle') + + return not stop_the_scroll diff --git a/elastalert/zabbix.py b/elastalert/zabbix.py new file mode 100644 index 000000000..e3f13aa03 --- /dev/null +++ b/elastalert/zabbix.py @@ -0,0 +1,75 @@ +from alerts import Alerter # , BasicMatchString +import logging +from pyzabbix.api import ZabbixAPI +from pyzabbix import ZabbixSender, ZabbixMetric +from datetime import datetime + + +class ZabbixClient(ZabbixAPI): + + def __init__(self, url='http://localhost', use_authenticate=False, user='Admin', password='zabbix', sender_host='localhost', + sender_port=10051): + self.url = url + self.use_authenticate = use_authenticate + self.sender_host = sender_host + self.sender_port = sender_port + self.metrics_chunk_size = 200 + self.aggregated_metrics = [] + self.logger = logging.getLogger(self.__class__.__name__) + super(ZabbixClient, self).__init__(url=self.url, use_authenticate=self.use_authenticate, user=user, password=password) + + def send_metric(self, hostname, key, data): + zm = ZabbixMetric(hostname, key, data) + if self.send_aggregated_metrics: + + self.aggregated_metrics.append(zm) + if len(self.aggregated_metrics) > self.metrics_chunk_size: + self.logger.info("Sending: %s metrics" % (len(self.aggregated_metrics))) + try: + ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port).send(self.aggregated_metrics) + self.aggregated_metrics = [] + except Exception as e: + self.logger.exception(e) + pass + else: + try: + ZabbixSender(zabbix_server=self.sender_host, zabbix_port=self.sender_port).send(zm) + except Exception as e: + self.logger.exception(e) + pass + + +class ZabbixAlerter(Alerter): + + # By setting required_options to a set of strings + # You can ensure that the rule config file specifies all + # of the options. Otherwise, ElastAlert will throw an exception + # when trying to load the rule. + required_options = frozenset(['zbx_sender_host', 'zbx_sender_port', 'zbx_host', 'zbx_key']) + + def __init__(self, *args): + super(ZabbixAlerter, self).__init__(*args) + + self.zbx_sender_host = self.rule.get('zbx_sender_host', 'localhost') + self.zbx_sender_port = self.rule.get('zbx_sender_port', 10051) + self.zbx_host = self.rule.get('zbx_host') + self.zbx_key = self.rule.get('zbx_key') + + # Alert is called + def alert(self, matches): + + # Matches is a list of match dictionaries. + # It contains more than one match when the alert has + # the aggregation option set + zm = [] + for match in matches: + ts_epoch = int(datetime.strptime(match['@timestamp'], "%Y-%m-%dT%H:%M:%S.%fZ").strftime('%s')) + zm.append(ZabbixMetric(host=self.zbx_host, key=self.zbx_key, value=1, clock=ts_epoch)) + + ZabbixSender(zabbix_server=self.zbx_sender_host, zabbix_port=self.zbx_sender_port).send(zm) + + # get_info is called after an alert is sent to get data that is written back + # to Elasticsearch in the field "alert_info" + # It should return a dict of information relevant to what the alert does + def get_info(self): + return {'type': 'zabbix Alerter'} diff --git a/example_rules/example_opsgenie_frequency.yaml b/example_rules/example_opsgenie_frequency.yaml index f8c835f46..9876f9162 100755 --- a/example_rules/example_opsgenie_frequency.yaml +++ b/example_rules/example_opsgenie_frequency.yaml @@ -21,11 +21,25 @@ opsgenie_key: ogkey #opsgenie_recipients: # - "neh" +# (Optional) +# OpsGenie recipients with args +# opsgenie_recipients: +# - {recipient} +# opsgenie_recipients_args: +# team_prefix:'user.email' + # (Optional) # OpsGenie teams to notify #opsgenie_teams: # - "Infrastructure" +# (Optional) +# OpsGenie teams with args +# opsgenie_teams: +# - {team_prefix}-Team +# opsgenie_teams_args: +# team_prefix:'team' + # (Optional) # OpsGenie alert tags opsgenie_tags: diff --git a/example_rules/example_spike.yaml b/example_rules/example_spike.yaml index 48e1f029d..cb7064c2e 100755 --- a/example_rules/example_spike.yaml +++ b/example_rules/example_spike.yaml @@ -31,7 +31,7 @@ index: logstash-* # (Required one of _cur or _ref, spike specific) # The minimum number of events that will trigger an alert # For example, if there are only 2 events between 12:00 and 2:00, and 20 between 2:00 and 4:00 -# _cur is 2 and _ref is 20, and the alert WILL fire because 20 is greater than threshold_cur +# _ref is 2 and _cur is 20, and the alert WILL fire because 20 is greater than threshold_cur and (_ref * spike_height) threshold_cur: 5 #threshold_ref: 5 diff --git a/example_rules/example_spike_single_metric_agg.yaml b/example_rules/example_spike_single_metric_agg.yaml new file mode 100644 index 000000000..b26ade15a --- /dev/null +++ b/example_rules/example_spike_single_metric_agg.yaml @@ -0,0 +1,55 @@ +name: Metricbeat Average CPU Spike Rule +type: spike_aggregation + +#es_host: localhost +#es_port: 9200 + +index: metricbeat-* + +timeframe: + hours: 4 + +buffer_time: + hours: 1 + +metric_agg_key: system.cpu.user.pct +metric_agg_type: avg +query_key: beat.hostname +doc_type: metricsets + +#allow_buffer_time_overlap: true +#use_run_every_query_size: true + +# (Required one of _cur or _ref, spike specific) +# The minimum value of the aggregation that will trigger the alert +# For example, if we're tracking the average for a metric whose average is 0.4 between 12:00 and 2:00 +# and 0.95 between 2:00 and 4:00 with spike_height set to 2 and threshhold_cur set to 0.9: +# _ref is 0.4 and _cur is 0.95, and the alert WILL fire +# because 0.95 is greater than threshold_cur (0.9) and (_ref * spike_height (.4 * 2)) +threshold_cur: 0.9 + +# (Optional, min_doc_count) +# for rules using a per-term aggregation via query_key, the minimum number of events +# over the past buffer_time needed to update the spike tracker +min_doc_count: 5 + +# (Required, spike specific) +# The spike aggregation rule matches when the current window contains spike_height times higher aggregated value +# than the reference window +spike_height: 2 + +# (Required, spike specific) +# The direction of the spike +# 'up' matches only spikes, 'down' matches only troughs +# 'both' matches both spikes and troughs +spike_type: "up" + +filter: +- term: + metricset.name: cpu + +# (Required) +# The alert is use when a match is found +alert: +- "debug" + diff --git a/example_rules/ssh-repeat-offender.yaml b/example_rules/ssh-repeat-offender.yaml new file mode 100644 index 000000000..27a439fcd --- /dev/null +++ b/example_rules/ssh-repeat-offender.yaml @@ -0,0 +1,61 @@ +# Rule name, must be unique +name: SSH abuse - reapeat offender + +# Alert on x events in y seconds +type: frequency + +# Alert when this many documents matching the query occur within a timeframe +num_events: 2 + +# num_events must occur within this amount of time to trigger an alert +timeframe: + weeks: 1 + +# A list of elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: + - term: + rule_name: "SSH abuse" + +index: elastalert + +# When the attacker continues, send a new alert after x minutes +realert: + weeks: 4 + +query_key: + - match_body.source.ip + +include: + - match_body.host.hostname + - match_body.user.name + - match_body.source.ip + +alert_subject: "SSH abuse (repeat offender) on <{}> | <{}|Show Dashboard>" +alert_subject_args: + - match_body.host.hostname + - kibana_link + +alert_text: |- + An reapeat offender has been active on {}. + + IP: {} + User: {} +alert_text_args: + - match_body.host.hostname + - match_body.user.name + - match_body.source.ip + +# The alert is use when a match is found +alert: + - slack + +slack_webhook_url: "https://hooks.slack.com/services/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" +slack_username_override: "ElastAlert" + +# Alert body only cointains a title and text +alert_text_type: alert_text_only + +# Link to BitSensor Kibana Dashboard +use_kibana4_dashboard: "https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb" diff --git a/example_rules/ssh.yaml b/example_rules/ssh.yaml new file mode 100644 index 000000000..7af890784 --- /dev/null +++ b/example_rules/ssh.yaml @@ -0,0 +1,64 @@ +# Rule name, must be unique + name: SSH abuse (ElastAlert 3.0.1) - 2 + +# Alert on x events in y seconds +type: frequency + +# Alert when this many documents matching the query occur within a timeframe +num_events: 20 + +# num_events must occur within this amount of time to trigger an alert +timeframe: + minutes: 60 + +# A list of elasticsearch filters used for find events +# These filters are joined with AND and nested in a filtered query +# For more info: http://www.elasticsearch.org/guide/en/elasticsearch/reference/current/query-dsl.html +filter: +- query: + query_string: + query: "event.type:authentication_failure" + +index: auditbeat-* + +# When the attacker continues, send a new alert after x minutes +realert: + minutes: 1 + +query_key: + - source.ip + +include: + - host.hostname + - user.name + - source.ip + +include_match_in_root: true + +alert_subject: "SSH abuse on <{}> | <{}|Show Dashboard>" +alert_subject_args: + - host.hostname + - kibana_link + +alert_text: |- + An attack on {} is detected. + The attacker looks like: + User: {} + IP: {} +alert_text_args: + - host.hostname + - user.name + - source.ip + +# The alert is use when a match is found +alert: + - debug + +slack_webhook_url: "https://hooks.slack.com/services/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" +slack_username_override: "ElastAlert" + +# Alert body only cointains a title and text +alert_text_type: alert_text_only + +# Link to BitSensor Kibana Dashboard +use_kibana4_dashboard: "https://dev.securely.ai/app/kibana#/dashboard/37739d80-a95c-11e9-b5ba-33a34ca252fb" diff --git a/pytest.ini b/pytest.ini new file mode 100644 index 000000000..0ad3341d9 --- /dev/null +++ b/pytest.ini @@ -0,0 +1,3 @@ +[pytest] +markers = + elasticsearch: mark a test as using elasticsearch. diff --git a/requirements-dev.txt b/requirements-dev.txt index 36daa0ebd..558761d9e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -1,4 +1,5 @@ -coverage +-r requirements.txt +coverage==4.5.4 flake8 pre-commit pylint<1.4 diff --git a/requirements.txt b/requirements.txt index 47676fc9b..9c32052d0 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,17 +1,21 @@ +apscheduler>=3.3.0 aws-requests-auth>=0.3.0 blist>=1.3.6 boto3>=1.4.4 +cffi>=1.11.5 configparser>=3.5.0 croniter>=0.3.16 -elasticsearch +elasticsearch>=7.0.0 envparse>=0.2.0 exotel>=0.1.3 jira>=1.0.10,<1.0.15 -jsonschema>=2.6.0 +jsonschema>=3.0.2 mock>=2.0.0 +prison>=0.1.2 +py-zabbix==1.1.3 PyStaticConfiguration>=0.10.3 python-dateutil>=2.6.0,<2.7.0 -PyYAML>=3.12 +PyYAML>=5.1 requests>=2.0.0 stomp.py>=4.1.17 texttable>=0.8.8 diff --git a/setup.py b/setup.py index 38562eec6..2845836a7 100644 --- a/setup.py +++ b/setup.py @@ -8,14 +8,14 @@ base_dir = os.path.dirname(__file__) setup( name='elastalert', - version='0.1.33', + version='0.2.4', description='Runs custom filters on Elasticsearch and alerts on matches', author='Quentin Long', author_email='qlo@yelp.com', setup_requires='setuptools', license='Copyright 2014 Yelp', classifiers=[ - 'Programming Language :: Python :: 2.7', + 'Programming Language :: Python :: 3.6', 'License :: OSI Approved :: Apache Software License', 'Operating System :: OS Independent', ], @@ -25,19 +25,21 @@ 'elastalert-rule-from-kibana=elastalert.rule_from_kibana:main', 'elastalert=elastalert.elastalert:main']}, packages=find_packages(), - package_data={'elastalert': ['schema.yaml']}, + package_data={'elastalert': ['schema.yaml', 'es_mappings/**/*.json']}, install_requires=[ + 'apscheduler>=3.3.0', 'aws-requests-auth>=0.3.0', 'blist>=1.3.6', 'boto3>=1.4.4', 'configparser>=3.5.0', 'croniter>=0.3.16', - 'elasticsearch', + 'elasticsearch==7.0.0', 'envparse>=0.2.0', 'exotel>=0.1.3', - 'jira>=1.0.10,<1.0.15', - 'jsonschema>=2.6.0', + 'jira>=2.0.0', + 'jsonschema>=3.0.2', 'mock>=2.0.0', + 'prison>=0.1.2', 'PyStaticConfiguration>=0.10.3', 'python-dateutil>=2.6.0,<2.7.0', 'PyYAML>=3.12', @@ -45,5 +47,6 @@ 'stomp.py>=4.1.17', 'texttable>=0.8.8', 'twilio>=6.0.0,<6.1', + 'cffi>=1.11.5' ] ) diff --git a/tests/alerts_test.py b/tests/alerts_test.py index 3271b511b..5cd61ae75 100644 --- a/tests/alerts_test.py +++ b/tests/alerts_test.py @@ -1,8 +1,8 @@ # -*- coding: utf-8 -*- +import base64 import datetime import json import subprocess -from contextlib import nested import mock import pytest @@ -21,7 +21,7 @@ from elastalert.alerts import PagerDutyAlerter from elastalert.alerts import SlackAlerter from elastalert.alerts import StrideAlerter -from elastalert.config import load_modules +from elastalert.loaders import FileRulesLoader from elastalert.opsgenie import OpsGenieAlerter from elastalert.util import ts_add from elastalert.util import ts_now @@ -35,7 +35,7 @@ def get_match_str(self, event): def test_basic_match_string(ea): ea.rules[0]['top_count_keys'] = ['username'] match = {'@timestamp': '1918-01-17', 'field': 'value', 'top_events_username': {'bob': 10, 'mallory': 5}} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'anytest' in alert_text assert 'some stuff happened' in alert_text assert 'username' in alert_text @@ -44,32 +44,32 @@ def test_basic_match_string(ea): # Non serializable objects don't cause errors match['non-serializable'] = {open: 10} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) # unicode objects dont cause errors - match['snowman'] = u'ā˜ƒ' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + match['snowman'] = 'ā˜ƒ' + alert_text = str(BasicMatchString(ea.rules[0], match)) # Pretty printed objects match.pop('non-serializable') match['object'] = {'this': {'that': [1, 2, "3"]}} - alert_text = unicode(BasicMatchString(ea.rules[0], match)) - assert '"this": {\n "that": [\n 1, \n 2, \n "3"\n ]\n }' in alert_text + alert_text = str(BasicMatchString(ea.rules[0], match)) + assert '"this": {\n "that": [\n 1,\n 2,\n "3"\n ]\n }' in alert_text ea.rules[0]['alert_text'] = 'custom text' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'anytest' not in alert_text ea.rules[0]['alert_text_type'] = 'alert_text_only' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'some stuff happened' not in alert_text assert 'username' not in alert_text assert 'field: value' not in alert_text ea.rules[0]['alert_text_type'] = 'exclude_fields' - alert_text = unicode(BasicMatchString(ea.rules[0], match)) + alert_text = str(BasicMatchString(ea.rules[0], match)) assert 'custom text' in alert_text assert 'some stuff happened' in alert_text assert 'username' in alert_text @@ -83,8 +83,8 @@ def test_jira_formatted_match_string(ea): expected_alert_text_snippet = '{code}{\n' \ + tab + '"foo": {\n' \ + 2 * tab + '"bar": [\n' \ - + 3 * tab + '"one", \n' \ - + 3 * tab + '2, \n' \ + + 3 * tab + '"one",\n' \ + + 3 * tab + '2,\n' \ + 3 * tab + '"three"\n' \ + 2 * tab + ']\n' \ + tab + '}\n' \ @@ -95,7 +95,7 @@ def test_jira_formatted_match_string(ea): def test_email(): rule = {'name': 'test alert', 'email': ['testing@test.test', 'test@test.test'], 'from_addr': 'testfrom@test.test', 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value', - 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': u'ā˜ƒ'} + 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': 'ā˜ƒ'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: mock_smtp.return_value = mock.Mock() @@ -106,7 +106,7 @@ def test_email(): mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -158,9 +158,9 @@ def test_email_from_field(): def test_email_with_unicode_strings(): - rule = {'name': 'test alert', 'email': u'testing@test.test', 'from_addr': 'testfrom@test.test', + rule = {'name': 'test alert', 'email': 'testing@test.test', 'from_addr': 'testfrom@test.test', 'type': mock_rule(), 'timestamp_field': '@timestamp', 'email_reply_to': 'test@example.com', 'owner': 'owner_value', - 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': u'ā˜ƒ'} + 'alert_subject': 'Test alert for {0}, owned by {1}', 'alert_subject_args': ['test_term', 'owner'], 'snowman': 'ā˜ƒ'} with mock.patch('elastalert.alerts.SMTP') as mock_smtp: mock_smtp.return_value = mock.Mock() @@ -170,8 +170,8 @@ def test_email_with_unicode_strings(): mock.call().ehlo(), mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), - mock.call().sendmail(mock.ANY, [u'testing@test.test'], mock.ANY), - mock.call().close()] + mock.call().sendmail(mock.ANY, ['testing@test.test'], mock.ANY), + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -200,7 +200,7 @@ def test_email_with_auth(): mock.call().starttls(certfile=None, keyfile=None), mock.call().login('someone', 'hunter2'), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected @@ -222,7 +222,7 @@ def test_email_with_cert_key(): mock.call().starttls(certfile='dummy/cert.crt', keyfile='dummy/client.key'), mock.call().login('someone', 'hunter2'), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected @@ -240,7 +240,7 @@ def test_email_with_cc(): mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test', 'tester@testing.testing'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -265,7 +265,7 @@ def test_email_with_bcc(): mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test', 'tester@testing.testing'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -300,7 +300,7 @@ def test_email_with_cc_and_bcc(): ], mock.ANY ), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] @@ -329,18 +329,18 @@ def test_email_with_args(): mock_smtp.return_value = mock.Mock() alert = EmailAlerter(rule) - alert.alert([{'test_term': 'test_value', 'test_arg1': 'testing', 'test': {'term': ':)', 'arg3': u'ā˜ƒ'}}]) + alert.alert([{'test_term': 'test_value', 'test_arg1': 'testing', 'test': {'term': ':)', 'arg3': 'ā˜ƒ'}}]) expected = [mock.call('localhost'), mock.call().ehlo(), mock.call().has_extn('STARTTLS'), mock.call().starttls(certfile=None, keyfile=None), mock.call().sendmail(mock.ANY, ['testing@test.test', 'test@test.test'], mock.ANY), - mock.call().close()] + mock.call().quit()] assert mock_smtp.mock_calls == expected body = mock_smtp.mock_calls[4][1][2] # Extract the MIME encoded message body - body_text = body.split('\n\n')[-1][:-1].decode('base64') + body_text = base64.b64decode(body.split('\n\n')[-1][:-1]).decode('utf-8') assert 'testing' in body_text assert '' in body_text @@ -380,16 +380,16 @@ def test_opsgenie_basic(): alert = OpsGenieAlerter(rule) alert.alert([{'@timestamp': '2014-10-31T00:00:00'}]) - print("mock_post: {0}".format(mock_post._mock_call_args_list)) + print(("mock_post: {0}".format(mock_post._mock_call_args_list))) mcal = mock_post._mock_call_args_list - print('mcal: {0}'.format(mcal[0])) + print(('mcal: {0}'.format(mcal[0]))) assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts') assert mock_post.called assert mcal[0][1]['headers']['Authorization'] == 'GenieKey ogkey' assert mcal[0][1]['json']['source'] == 'ElastAlert' - assert mcal[0][1]['json']['responders'] == [{'id': 'lytics', 'type': 'user'}] + assert mcal[0][1]['json']['responders'] == [{'username': 'lytics', 'type': 'user'}] assert mcal[0][1]['json']['source'] == 'ElastAlert' @@ -406,20 +406,311 @@ def test_opsgenie_frequency(): assert alert.get_info()['recipients'] == rule['opsgenie_recipients'] - print("mock_post: {0}".format(mock_post._mock_call_args_list)) + print(("mock_post: {0}".format(mock_post._mock_call_args_list))) mcal = mock_post._mock_call_args_list - print('mcal: {0}'.format(mcal[0])) + print(('mcal: {0}'.format(mcal[0]))) assert mcal[0][0][0] == ('https://api.opsgenie.com/v2/alerts') assert mock_post.called assert mcal[0][1]['headers']['Authorization'] == 'GenieKey ogkey' assert mcal[0][1]['json']['source'] == 'ElastAlert' - assert mcal[0][1]['json']['responders'] == [{'id': 'lytics', 'type': 'user'}] + assert mcal[0][1]['json']['responders'] == [{'username': 'lytics', 'type': 'user'}] assert mcal[0][1]['json']['source'] == 'ElastAlert' assert mcal[0][1]['json']['source'] == 'ElastAlert' +def test_opsgenie_alert_routing(): + rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey', + 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts', + 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX': 'recipient'}, + 'type': mock_rule(), + 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}], + 'alert': 'opsgenie', + 'opsgenie_teams': ['{TEAM_PREFIX}-Team'], 'opsgenie_teams_args': {'TEAM_PREFIX': 'team'}} + with mock.patch('requests.post'): + + alert = OpsGenieAlerter(rule) + alert.alert([{'@timestamp': '2014-10-31T00:00:00', 'team': "Test", 'recipient': "lytics"}]) + + assert alert.get_info()['teams'] == ['Test-Team'] + assert alert.get_info()['recipients'] == ['lytics'] + + +def test_opsgenie_default_alert_routing(): + rule = {'name': 'testOGalert', 'opsgenie_key': 'ogkey', + 'opsgenie_account': 'genies', 'opsgenie_addr': 'https://api.opsgenie.com/v2/alerts', + 'opsgenie_recipients': ['{RECEIPIENT_PREFIX}'], 'opsgenie_recipients_args': {'RECEIPIENT_PREFIX': 'recipient'}, + 'type': mock_rule(), + 'filter': [{'query': {'query_string': {'query': '*hihi*'}}}], + 'alert': 'opsgenie', + 'opsgenie_teams': ['{TEAM_PREFIX}-Team'], + 'opsgenie_default_receipients': ["devops@test.com"], 'opsgenie_default_teams': ["Test"] + } + with mock.patch('requests.post'): + + alert = OpsGenieAlerter(rule) + alert.alert([{'@timestamp': '2014-10-31T00:00:00', 'team': "Test"}]) + + assert alert.get_info()['teams'] == ['{TEAM_PREFIX}-Team'] + assert alert.get_info()['recipients'] == ['devops@test.com'] + + +def test_opsgenie_details_with_constant_value(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': 'Bar'} + } + match = { + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': {'field': 'message'}} + } + match = { + 'message': 'Bar', + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_nested_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': {'field': 'nested.field'}} + } + match = { + 'nested': { + 'field': 'Bar' + }, + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_non_string_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': { + 'Age': {'field': 'age'}, + 'Message': {'field': 'message'} + } + } + match = { + 'age': 10, + 'message': { + 'format': 'The cow goes %s!', + 'arg0': 'moo' + } + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': { + 'Age': '10', + 'Message': "{'format': 'The cow goes %s!', 'arg0': 'moo'}" + }, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_missing_field(): + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': { + 'Message': {'field': 'message'}, + 'Missing': {'field': 'missing'} + } + } + match = { + 'message': 'Testing', + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Message': 'Testing'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + +def test_opsgenie_details_with_environment_variable_replacement(environ): + environ.update({ + 'TEST_VAR': 'Bar' + }) + rule = { + 'name': 'Opsgenie Details', + 'type': mock_rule(), + 'opsgenie_account': 'genies', + 'opsgenie_key': 'ogkey', + 'opsgenie_details': {'Foo': '$TEST_VAR'} + } + match = { + '@timestamp': '2014-10-31T00:00:00' + } + alert = OpsGenieAlerter(rule) + + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + mock_post_request.assert_called_once_with( + 'https://api.opsgenie.com/v2/alerts', + headers={ + 'Content-Type': 'application/json', + 'Authorization': 'GenieKey ogkey' + }, + json=mock.ANY, + proxies=None + ) + + expected_json = { + 'description': BasicMatchString(rule, match).__str__(), + 'details': {'Foo': 'Bar'}, + 'message': 'ElastAlert: Opsgenie Details', + 'priority': None, + 'source': 'ElastAlert', + 'tags': ['ElastAlert', 'Opsgenie Details'], + 'user': 'genies' + } + actual_json = mock_post_request.call_args_list[0][1]['json'] + assert expected_json == actual_json + + def test_jira(): description_txt = "Description stuff goes here like a runbook link." rule = { @@ -442,10 +733,8 @@ def test_jira(): mock_priority = mock.Mock(id='5') - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = [] @@ -475,10 +764,8 @@ def test_jira(): # Search called if jira_bump_tickets rule['jira_bump_tickets'] = True - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -493,10 +780,8 @@ def test_jira(): # Remove a field if jira_ignore_in_title set rule['jira_ignore_in_title'] = 'test_term' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [] @@ -509,10 +794,8 @@ def test_jira(): assert 'test_value' not in mock_jira.mock_calls[3][1][0] # Issue is still created if search_issues throws an exception - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.side_effect = JIRAError @@ -530,10 +813,8 @@ def test_jira(): # Check ticket is bumped if it is updated 4 days ago mock_issue.fields.updated = str(ts_now() - datetime.timedelta(days=4)) - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -548,10 +829,8 @@ def test_jira(): # Check ticket is bumped is not bumped if ticket is updated right now mock_issue.fields.updated = str(ts_now()) - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -585,10 +864,8 @@ def test_jira(): mock_fields = [ {'name': 'affected user', 'id': 'affected_user_id', 'schema': {'type': 'string'}} ] - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value = mock.Mock() mock_jira.return_value.search_issues.return_value = [mock_issue] @@ -660,10 +937,8 @@ def test_jira_arbitrary_field_support(): }, ] - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -703,10 +978,8 @@ def test_jira_arbitrary_field_support(): # Reference an arbitrary string field that is not defined on the JIRA server rule['jira_nonexistent_field'] = 'nonexistent field value' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -721,10 +994,8 @@ def test_jira_arbitrary_field_support(): # Reference a watcher that does not exist rule['jira_watchers'] = 'invalid_watcher' - with nested( - mock.patch('elastalert.alerts.JIRA'), - mock.patch('elastalert.alerts.yaml_loader') - ) as (mock_jira, mock_open): + with mock.patch('elastalert.alerts.JIRA') as mock_jira, \ + mock.patch('elastalert.alerts.yaml_loader') as mock_open: mock_open.return_value = {'user': 'jirauser', 'password': 'jirapassword'} mock_jira.return_value.priorities.return_value = [mock_priority] mock_jira.return_value.fields.return_value = mock_fields @@ -866,7 +1137,8 @@ def test_ms_teams(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = MsTeamsAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -902,7 +1174,8 @@ def test_ms_teams_uses_color_and_fixed_width_text(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = MsTeamsAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -938,7 +1211,8 @@ def test_slack_uses_custom_title(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -967,7 +1241,55 @@ def test_slack_uses_custom_title(): rule['slack_webhook_url'], data=mock.ANY, headers={'content-type': 'application/json'}, - proxies=None + proxies=None, + verify=False, + timeout=10 + ) + assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) + + +def test_slack_uses_custom_timeout(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert_subject': 'Cool subject', + 'alert': [], + 'slack_timeout': 20 + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'somefield': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'channel': '', + 'icon_emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': rule['alert_subject'], + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'text': '', + 'parse': 'none' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=20 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -979,7 +1301,8 @@ def test_slack_uses_rule_name_when_custom_title_is_not_provided(): 'slack_webhook_url': ['http://please.dontgohere.slack'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1002,13 +1325,15 @@ def test_slack_uses_rule_name_when_custom_title_is_not_provided(): } ], 'text': '', - 'parse': 'none' + 'parse': 'none', } mock_post_request.assert_called_once_with( rule['slack_webhook_url'][0], data=mock.ANY, headers={'content-type': 'application/json'}, - proxies=None + proxies=None, + verify=False, + timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) @@ -1021,7 +1346,8 @@ def test_slack_uses_custom_slack_channel(): 'slack_channel_override': '#test-alert', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = SlackAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1044,17 +1370,281 @@ def test_slack_uses_custom_slack_channel(): } ], 'text': '', - 'parse': 'none' + 'parse': 'none', } mock_post_request.assert_called_once_with( rule['slack_webhook_url'][0], data=mock.ANY, headers={'content-type': 'application/json'}, - proxies=None + proxies=None, + verify=False, + timeout=10 ) assert expected_data == json.loads(mock_post_request.call_args_list[0][1]['data']) +def test_slack_uses_list_of_custom_slack_channel(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_webhook_url': ['http://please.dontgohere.slack'], + 'slack_channel_override': ['#test-alert', '#test-alert2'], + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'somefield': 'foobarbaz' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data1 = { + 'username': 'elastalert', + 'channel': '#test-alert', + 'icon_emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': rule['name'], + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'text': '', + 'parse': 'none' + } + expected_data2 = { + 'username': 'elastalert', + 'channel': '#test-alert2', + 'icon_emoji': ':ghost:', + 'attachments': [ + { + 'color': 'danger', + 'title': rule['name'], + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'text': '', + 'parse': 'none' + } + mock_post_request.assert_called_with( + rule['slack_webhook_url'][0], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + assert expected_data1 == json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data2 == json.loads(mock_post_request.call_args_list[1][1]['data']) + + +def test_slack_attach_kibana_discover_url_when_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_attach_kibana_discover_url_when_not_generated(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_kibana_discover_title(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_kibana_discover_title': 'Click to discover in Kibana', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': '#ec4b98', + 'title': 'Click to discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + +def test_slack_kibana_discover_color(): + rule = { + 'name': 'Test Rule', + 'type': 'any', + 'slack_attach_kibana_discover_url': True, + 'slack_kibana_discover_color': 'blue', + 'slack_webhook_url': 'http://please.dontgohere.slack', + 'alert': [] + } + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) + alert = SlackAlerter(rule) + match = { + '@timestamp': '2016-01-01T00:00:00', + 'kibana_discover_url': 'http://kibana#discover' + } + with mock.patch('requests.post') as mock_post_request: + alert.alert([match]) + + expected_data = { + 'username': 'elastalert', + 'parse': 'none', + 'text': '', + 'attachments': [ + { + 'color': 'danger', + 'title': 'Test Rule', + 'text': BasicMatchString(rule, match).__str__(), + 'mrkdwn_in': ['text', 'pretext'], + 'fields': [] + }, + { + 'color': 'blue', + 'title': 'Discover in Kibana', + 'title_link': 'http://kibana#discover' + } + ], + 'icon_emoji': ':ghost:', + 'channel': '' + } + mock_post_request.assert_called_once_with( + rule['slack_webhook_url'], + data=mock.ANY, + headers={'content-type': 'application/json'}, + proxies=None, + verify=False, + timeout=10 + ) + actual_data = json.loads(mock_post_request.call_args_list[0][1]['data']) + assert expected_data == actual_data + + def test_http_alerter_with_payload(): rule = { 'name': 'Test HTTP Post Alerter With Payload', @@ -1064,7 +1654,8 @@ def test_http_alerter_with_payload(): 'http_post_static_payload': {'name': 'somestaticname'}, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1096,7 +1687,8 @@ def test_http_alerter_with_payload_all_values(): 'http_post_all_values': True, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1128,7 +1720,8 @@ def test_http_alerter_without_payload(): 'http_post_static_payload': {'name': 'somestaticname'}, 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HTTPPostAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1159,7 +1752,8 @@ def test_pagerduty_alerter(): 'pagerduty_client_name': 'ponies inc.', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1196,7 +1790,8 @@ def test_pagerduty_alerter_v2(): 'pagerduty_v2_payload_source': 'mysql.host.name', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1216,6 +1811,7 @@ def test_pagerduty_alerter_v2(): 'custom_details': { 'information': 'Test PD Rule\n\n@timestamp: 2017-01-01T00:00:00\nsomefield: foobarbaz\n' }, + 'timestamp': '2017-01-01T00:00:00' }, 'event_action': 'trigger', 'dedup_key': '', @@ -1235,7 +1831,8 @@ def test_pagerduty_alerter_custom_incident_key(): 'pagerduty_incident_key': 'custom key', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1267,7 +1864,8 @@ def test_pagerduty_alerter_custom_incident_key_with_args(): 'pagerduty_incident_key_args': ['somefield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1300,7 +1898,8 @@ def test_pagerduty_alerter_custom_alert_subject(): 'pagerduty_incident_key_args': ['somefield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1334,7 +1933,8 @@ def test_pagerduty_alerter_custom_alert_subject_with_args(): 'pagerduty_incident_key_args': ['someotherfield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1370,7 +1970,8 @@ def test_pagerduty_alerter_custom_alert_subject_with_args_specifying_trigger(): 'pagerduty_incident_key_args': ['someotherfield'], 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = PagerDutyAlerter(rule) match = { '@timestamp': '2017-01-01T00:00:00', @@ -1401,7 +2002,7 @@ def test_alert_text_kw(ea): 'field': 'field', } match = {'@timestamp': '1918-01-17', 'field': 'value'} - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) body = '{field} at {@timestamp}'.format(**match) assert body in alert_text @@ -1420,7 +2021,7 @@ def test_alert_text_global_substitution(ea): 'abc': 'abc from match', } - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) assert 'Priority: priority from rule' in alert_text assert 'Owner: the owner from rule' in alert_text @@ -1446,7 +2047,7 @@ def test_alert_text_kw_global_substitution(ea): 'abc': 'abc from match', } - alert_text = unicode(BasicMatchString(rule, match)) + alert_text = str(BasicMatchString(rule, match)) assert 'Owner: the owner from rule' in alert_text assert 'Foo: foo from rule' in alert_text @@ -1495,7 +2096,8 @@ def test_stride_plain_text(): 'alert_subject': 'Cool subject', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1540,7 +2142,8 @@ def test_stride_underline_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1585,7 +2188,8 @@ def test_stride_bold_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1630,7 +2234,8 @@ def test_stride_strong_text(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1675,7 +2280,8 @@ def test_stride_hyperlink(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1720,7 +2326,8 @@ def test_stride_html(): 'alert_text_type': 'alert_text_only', 'alert': [] } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = StrideAlerter(rule) match = { '@timestamp': '2016-01-01T00:00:00', @@ -1772,7 +2379,8 @@ def test_hipchat_body_size_limit_text(): 'message': 'message', }, } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HipChatAlerter(rule) match = { '@timestamp': '2018-01-01T00:00:00', @@ -1799,7 +2407,8 @@ def test_hipchat_body_size_limit_html(): 'message': 'message', }, } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = HipChatAlerter(rule) match = { '@timestamp': '2018-01-01T00:00:00', @@ -1816,7 +2425,8 @@ def test_alerta_no_auth(ea): 'name': 'Test Alerta rule!', 'alerta_api_url': 'http://elastalerthost:8080/api/alert', 'timeframe': datetime.timedelta(hours=1), - 'timestamp_field': u'@timestamp', + 'timestamp_field': '@timestamp', + 'alerta_api_skip_ssl': True, 'alerta_attributes_keys': ["hostname", "TimestampEvent", "senderIP"], 'alerta_attributes_values': ["%(key)s", "%(logdate)s", "%(sender_ip)s"], 'alerta_correlate': ["ProbeUP", "ProbeDOWN"], @@ -1832,14 +2442,15 @@ def test_alerta_no_auth(ea): } match = { - u'@timestamp': '2014-10-10T00:00:00', + '@timestamp': '2014-10-10T00:00:00', # 'key': ---- missing field on purpose, to verify that simply the text is left empty # 'logdate': ---- missing field on purpose, to verify that simply the text is left empty 'sender_ip': '1.1.1.1', 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) @@ -1867,7 +2478,8 @@ def test_alerta_no_auth(ea): alert.url, data=mock.ANY, headers={ - 'content-type': 'application/json'} + 'content-type': 'application/json'}, + verify=False ) assert expected_data == json.loads( mock_post_request.call_args_list[0][1]['data']) @@ -1892,7 +2504,8 @@ def test_alerta_auth(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) @@ -1900,6 +2513,7 @@ def test_alerta_auth(ea): mock_post_request.assert_called_once_with( alert.url, data=mock.ANY, + verify=True, headers={ 'content-type': 'application/json', 'Authorization': 'Key {}'.format(rule['alerta_api_key'])}) @@ -1934,7 +2548,8 @@ def test_alerta_new_style(ea): 'hostname': 'aProbe' } - load_modules(rule) + rules_loader = FileRulesLoader({}) + rules_loader.load_modules(rule) alert = AlertaAlerter(rule) with mock.patch('requests.post') as mock_post_request: alert.alert([match]) @@ -1961,8 +2576,38 @@ def test_alerta_new_style(ea): mock_post_request.assert_called_once_with( alert.url, data=mock.ANY, + verify=True, headers={ 'content-type': 'application/json'} ) assert expected_data == json.loads( mock_post_request.call_args_list[0][1]['data']) + + +def test_alert_subject_size_limit_no_args(ea): + rule = { + 'name': 'test_rule', + 'type': mock_rule(), + 'owner': 'the_owner', + 'priority': 2, + 'alert_subject': 'A very long subject', + 'alert_subject_max_len': 5 + } + alert = Alerter(rule) + alertSubject = alert.create_custom_title([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}]) + assert 5 == len(alertSubject) + + +def test_alert_subject_size_limit_with_args(ea): + rule = { + 'name': 'test_rule', + 'type': mock_rule(), + 'owner': 'the_owner', + 'priority': 2, + 'alert_subject': 'Test alert for {0} {1}', + 'alert_subject_args': ['test_term', 'test.term'], + 'alert_subject_max_len': 6 + } + alert = Alerter(rule) + alertSubject = alert.create_custom_title([{'test_term': 'test_value', '@timestamp': '2014-10-31T00:00:00'}]) + assert 6 == len(alertSubject) diff --git a/tests/base_test.py b/tests/base_test.py index b10eb5a74..92dc35f7e 100644 --- a/tests/base_test.py +++ b/tests/base_test.py @@ -1,5 +1,4 @@ # -*- coding: utf-8 -*- -import contextlib import copy import datetime import json @@ -22,7 +21,6 @@ from elastalert.util import ts_to_dt from elastalert.util import unix_to_dt - START_TIMESTAMP = '2014-09-26T12:34:45Z' END_TIMESTAMP = '2014-09-27T12:34:45Z' START = ts_to_dt(START_TIMESTAMP) @@ -41,7 +39,7 @@ def generate_hits(timestamps, **kwargs): '_source': {'@timestamp': ts}, '_type': 'logs', '_index': 'idx'} - for key, item in kwargs.iteritems(): + for key, item in kwargs.items(): data['_source'][key] = item # emulate process_hits(), add metadata to _source for field in ['_id', '_type', '_index']: @@ -71,13 +69,13 @@ def test_init_rule(ea): # Simulate state of a rule just loaded from a file ea.rules[0]['minimum_starttime'] = datetime.datetime.now() new_rule = copy.copy(ea.rules[0]) - map(new_rule.pop, ['agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime']) + list(map(new_rule.pop, ['agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime'])) # Properties are copied from ea.rules[0] ea.rules[0]['starttime'] = '2014-01-02T00:11:22' ea.rules[0]['processed_hits'] = ['abcdefg'] new_rule = ea.init_rule(new_rule, False) - for prop in ['starttime', 'agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime']: + for prop in ['starttime', 'agg_matches', 'current_aggregate_id', 'processed_hits', 'minimum_starttime', 'run_every']: assert new_rule[prop] == ea.rules[0][prop] # Properties are fresh @@ -86,54 +84,119 @@ def test_init_rule(ea): assert 'starttime' not in new_rule assert new_rule['processed_hits'] == {} + # Assert run_every is unique + new_rule['run_every'] = datetime.timedelta(seconds=17) + new_rule = ea.init_rule(new_rule, True) + assert new_rule['run_every'] == datetime.timedelta(seconds=17) + def test_query(ea): - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) - ea.current_es.search.assert_called_with(body={ - 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, + ea.thread_data.current_es.search.assert_called_with(body={ + 'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], + ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) +def test_query_sixsix(ea_sixsix): + ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.run_query(ea_sixsix.rules[0], START, END) + ea_sixsix.thread_data.current_es.search.assert_called_with(body={ + 'query': {'bool': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], + ignore_unavailable=True, + size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) + + def test_query_with_fields(ea): ea.rules[0]['_source_enabled'] = False - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) - ea.current_es.search.assert_called_with(body={ - 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, + ea.thread_data.current_es.search.assert_called_with(body={ + 'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, 'sort': [{'@timestamp': {'order': 'asc'}}], 'fields': ['@timestamp']}, index='idx', ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) +def test_query_sixsix_with_fields(ea_sixsix): + ea_sixsix.rules[0]['_source_enabled'] = False + ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.run_query(ea_sixsix.rules[0], START, END) + ea_sixsix.thread_data.current_es.search.assert_called_with(body={ + 'query': {'bool': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}], 'stored_fields': ['@timestamp']}, index='idx', + ignore_unavailable=True, + size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) + + def test_query_with_unix(ea): ea.rules[0]['timestamp_type'] = 'unix' ea.rules[0]['dt_to_ts'] = dt_to_unix - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) start_unix = dt_to_unix(START) end_unix = dt_to_unix(END) - ea.current_es.search.assert_called_with( - body={'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, + ea.thread_data.current_es.search.assert_called_with( + body={'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], + ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) +def test_query_sixsix_with_unix(ea_sixsix): + ea_sixsix.rules[0]['timestamp_type'] = 'unix' + ea_sixsix.rules[0]['dt_to_ts'] = dt_to_unix + ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.run_query(ea_sixsix.rules[0], START, END) + start_unix = dt_to_unix(START) + end_unix = dt_to_unix(END) + ea_sixsix.thread_data.current_es.search.assert_called_with( + body={'query': {'bool': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], + ignore_unavailable=True, + size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) + + def test_query_with_unixms(ea): ea.rules[0]['timestamp_type'] = 'unixms' ea.rules[0]['dt_to_ts'] = dt_to_unixms - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) start_unix = dt_to_unixms(START) end_unix = dt_to_unixms(END) - ea.current_es.search.assert_called_with( - body={'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, - 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], ignore_unavailable=True, + ea.thread_data.current_es.search.assert_called_with( + body={'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], + ignore_unavailable=True, size=ea.rules[0]['max_query_size'], scroll=ea.conf['scroll_keepalive']) +def test_query_sixsix_with_unixms(ea_sixsix): + ea_sixsix.rules[0]['timestamp_type'] = 'unixms' + ea_sixsix.rules[0]['dt_to_ts'] = dt_to_unixms + ea_sixsix.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea_sixsix.run_query(ea_sixsix.rules[0], START, END) + start_unix = dt_to_unixms(START) + end_unix = dt_to_unixms(END) + ea_sixsix.thread_data.current_es.search.assert_called_with( + body={'query': {'bool': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': end_unix, 'gt': start_unix}}}]}}}}, + 'sort': [{'@timestamp': {'order': 'asc'}}]}, index='idx', _source_include=['@timestamp'], + ignore_unavailable=True, + size=ea_sixsix.rules[0]['max_query_size'], scroll=ea_sixsix.conf['scroll_keepalive']) + + def test_no_hits(ea): - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 0 @@ -142,7 +205,7 @@ def test_no_terms_hits(ea): ea.rules[0]['use_terms_query'] = True ea.rules[0]['query_key'] = 'QWERTY' ea.rules[0]['doc_type'] = 'uiop' - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.deprecated_search.return_value = {'hits': {'total': 0, 'hits': []}} ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_terms_data.call_count == 0 @@ -150,7 +213,7 @@ def test_no_terms_hits(ea): def test_some_hits(ea): hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) hits_dt = generate_hits([START, END]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 1 ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']]) @@ -162,7 +225,7 @@ def test_some_hits_unix(ea): ea.rules[0]['ts_to_dt'] = unix_to_dt hits = generate_hits([dt_to_unix(START), dt_to_unix(END)]) hits_dt = generate_hits([START, END]) - ea.current_es.search.return_value = copy.deepcopy(hits) + ea.thread_data.current_es.search.return_value = copy.deepcopy(hits) ea.run_query(ea.rules[0], START, END) assert ea.rules[0]['type'].add_data.call_count == 1 ea.rules[0]['type'].add_data.assert_called_with([x['_source'] for x in hits_dt['hits']['hits']]) @@ -176,7 +239,7 @@ def _duplicate_hits_generator(timestamps, **kwargs): def test_duplicate_timestamps(ea): - ea.current_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP] * 3, blah='duplicate') + ea.thread_data.current_es.search.side_effect = _duplicate_hits_generator([START_TIMESTAMP] * 3, blah='duplicate') ea.run_query(ea.rules[0], START, ts_to_dt('2014-01-01T00:00:00Z')) assert len(ea.rules[0]['type'].add_data.call_args_list[0][0][0]) == 3 @@ -189,7 +252,7 @@ def test_duplicate_timestamps(ea): def test_match(ea): hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.run_rule(ea.rules[0], END, START) @@ -203,8 +266,8 @@ def test_run_rule_calls_garbage_collect(ea): end_time = '2014-09-26T12:00:00Z' ea.buffer_time = datetime.timedelta(hours=1) ea.run_every = datetime.timedelta(hours=1) - with contextlib.nested(mock.patch.object(ea.rules[0]['type'], 'garbage_collect'), - mock.patch.object(ea, 'run_query')) as (mock_gc, mock_get_hits): + with mock.patch.object(ea.rules[0]['type'], 'garbage_collect') as mock_gc, \ + mock.patch.object(ea, 'run_query'): ea.run_rule(ea.rules[0], ts_to_dt(end_time), ts_to_dt(start_time)) # Running ElastAlert every hour for 12 hours, we should see self.garbage_collect called 12 times. @@ -259,16 +322,16 @@ def test_match_with_module_from_pending(ea): pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} # First call, return the pending alert, second, no associated aggregated alerts - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, - {'hits': {'hits': []}}] + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, + {'hits': {'hits': []}}] ea.send_pending_alerts() assert mod.process.call_count == 0 # If aggregation is set, enhancement IS called pending_alert = {'match_body': {'foo': 'bar'}, 'rule_name': ea.rules[0]['name'], 'alert_time': START_TIMESTAMP, '@timestamp': START_TIMESTAMP} - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': pending_alert}]}}, - {'hits': {'hits': []}}] + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': pending_alert}]}}, + {'hits': {'hits': []}}] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.send_pending_alerts() assert mod.process.call_count == 1 @@ -280,7 +343,7 @@ def test_match_with_module_with_agg(ea): ea.rules[0]['match_enhancements'] = [mod] ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15) hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.run_rule(ea.rules[0], END, START) @@ -294,7 +357,7 @@ def test_match_with_enhancements_first(ea): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=15) ea.rules[0]['run_enhancements_first'] = True hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['type'].matches = [{'@timestamp': END}] with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch.object(ea, 'add_aggregated_alert') as add_alert: @@ -317,9 +380,10 @@ def test_agg_matchtime(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] alerttime1 = dt_to_ts(ts_to_dt(hits_timestamps[0]) + datetime.timedelta(minutes=10)) hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits - with mock.patch('elastalert.elastalert.elasticsearch_client'): + ea.thread_data.current_es.search.return_value = hits + with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: # Aggregate first two, query over full range + mock_es.return_value = ea.thread_data.current_es ea.rules[0]['aggregate_by_match_time'] = True ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps] @@ -345,10 +409,10 @@ def test_agg_matchtime(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call3}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call2}]}}, - {'hits': {'total': 0, 'hits': []}}] + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call3}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call2}]}}, + {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: ea.send_pending_alerts() @@ -357,15 +421,15 @@ def test_agg_matchtime(ea): assert mock_es.call_count == 2 assert_alerts(ea, [hits_timestamps[:2], hits_timestamps[2:]]) - call1 = ea.writeback_es.search.call_args_list[7][1]['body'] - call2 = ea.writeback_es.search.call_args_list[8][1]['body'] - call3 = ea.writeback_es.search.call_args_list[9][1]['body'] - call4 = ea.writeback_es.search.call_args_list[10][1]['body'] + call1 = ea.writeback_es.deprecated_search.call_args_list[7][1]['body'] + call2 = ea.writeback_es.deprecated_search.call_args_list[8][1]['body'] + call3 = ea.writeback_es.deprecated_search.call_args_list[9][1]['body'] + call4 = ea.writeback_es.deprecated_search.call_args_list[10][1]['body'] assert 'alert_time' in call2['filter']['range'] assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD' assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF' - assert ea.writeback_es.search.call_args_list[9][1]['size'] == 1337 + assert ea.writeback_es.deprecated_search.call_args_list[9][1]['size'] == 1337 def test_agg_not_matchtime(ea): @@ -373,7 +437,7 @@ def test_agg_not_matchtime(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] match_time = ts_to_dt('2014-09-26T12:55:00Z') hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch('elastalert.elastalert.ts_now', return_value=match_time): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) @@ -402,14 +466,15 @@ def test_agg_cron(ea): ea.max_aggregation = 1337 hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45'] hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits alerttime1 = dt_to_ts(ts_to_dt('2014-09-26T12:46:00')) alerttime2 = dt_to_ts(ts_to_dt('2014-09-26T13:04:00')) with mock.patch('elastalert.elastalert.elasticsearch_client'): with mock.patch('elastalert.elastalert.croniter.get_next') as mock_ts: # Aggregate first two, query over full range - mock_ts.side_effect = [dt_to_unix(ts_to_dt('2014-09-26T12:46:00')), dt_to_unix(ts_to_dt('2014-09-26T13:04:00'))] + mock_ts.side_effect = [dt_to_unix(ts_to_dt('2014-09-26T12:46:00')), + dt_to_unix(ts_to_dt('2014-09-26T13:04:00'))] ea.rules[0]['aggregation'] = {'schedule': '*/5 * * * *'} ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps] ea.run_rule(ea.rules[0], END, START) @@ -439,7 +504,7 @@ def test_agg_no_writeback_connectivity(ea): run again, that they will be passed again to add_aggregated_alert """ hit1, hit2, hit3 = '2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:47:45' hits = generate_hits([hit1, hit2, hit3]) - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': hit1}, {'@timestamp': hit2}, @@ -453,10 +518,10 @@ def test_agg_no_writeback_connectivity(ea): {'@timestamp': hit2, 'num_hits': 0, 'num_matches': 3}, {'@timestamp': hit3, 'num_hits': 0, 'num_matches': 3}] - ea.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + ea.thread_data.current_es.search.return_value = {'hits': {'total': 0, 'hits': []}} ea.add_aggregated_alert = mock.Mock() - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch.object(ea, 'run_query'): ea.run_rule(ea.rules[0], END, START) ea.add_aggregated_alert.assert_any_call({'@timestamp': hit1, 'num_hits': 0, 'num_matches': 3}, ea.rules[0]) @@ -469,8 +534,9 @@ def test_agg_with_aggregation_key(ea): hits_timestamps = ['2014-09-26T12:34:45', '2014-09-26T12:40:45', '2014-09-26T12:43:45'] match_time = ts_to_dt('2014-09-26T12:45:00Z') hits = generate_hits(hits_timestamps) - ea.current_es.search.return_value = hits - with mock.patch('elastalert.elastalert.elasticsearch_client'): + ea.thread_data.current_es.search.return_value = hits + with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: + mock_es.return_value = ea.thread_data.current_es with mock.patch('elastalert.elastalert.ts_now', return_value=match_time): ea.rules[0]['aggregation'] = datetime.timedelta(minutes=10) ea.rules[0]['type'].matches = [{'@timestamp': h} for h in hits_timestamps] @@ -511,27 +577,28 @@ def test_agg_with_aggregation_key(ea): # First call - Find all pending alerts (only entries without agg_id) # Second call - Find matches with agg_id == 'ABCD' # Third call - Find matches with agg_id == 'CDEF' - ea.writeback_es.search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_source': call1}, - {'_id': 'CDEF', '_source': call2}]}}, - {'hits': {'hits': [{'_id': 'BCDE', '_source': call3}]}}, - {'hits': {'total': 0, 'hits': []}}] + ea.writeback_es.deprecated_search.side_effect = [{'hits': {'hits': [{'_id': 'ABCD', '_index': 'wb', '_source': call1}, + {'_id': 'CDEF', '_index': 'wb', '_source': call2}]}}, + {'hits': {'hits': [{'_id': 'BCDE', '_index': 'wb', '_source': call3}]}}, + {'hits': {'total': 0, 'hits': []}}] with mock.patch('elastalert.elastalert.elasticsearch_client') as mock_es: + mock_es.return_value = ea.thread_data.current_es ea.send_pending_alerts() # Assert that current_es was refreshed from the aggregate rules assert mock_es.called_with(host='', port='') assert mock_es.call_count == 2 assert_alerts(ea, [[hits_timestamps[0], hits_timestamps[2]], [hits_timestamps[1]]]) - call1 = ea.writeback_es.search.call_args_list[7][1]['body'] - call2 = ea.writeback_es.search.call_args_list[8][1]['body'] - call3 = ea.writeback_es.search.call_args_list[9][1]['body'] - call4 = ea.writeback_es.search.call_args_list[10][1]['body'] + call1 = ea.writeback_es.deprecated_search.call_args_list[7][1]['body'] + call2 = ea.writeback_es.deprecated_search.call_args_list[8][1]['body'] + call3 = ea.writeback_es.deprecated_search.call_args_list[9][1]['body'] + call4 = ea.writeback_es.deprecated_search.call_args_list[10][1]['body'] assert 'alert_time' in call2['filter']['range'] assert call3['query']['query_string']['query'] == 'aggregate_id:ABCD' assert call4['query']['query_string']['query'] == 'aggregate_id:CDEF' - assert ea.writeback_es.search.call_args_list[9][1]['size'] == 1337 + assert ea.writeback_es.deprecated_search.call_args_list[9][1]['size'] == 1337 def test_silence(ea): @@ -561,12 +628,12 @@ def test_silence(ea): def test_compound_query_key(ea): ea.rules[0]['query_key'] = 'this,that,those' ea.rules[0]['compound_query_key'] = ['this', 'that', 'those'] - hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that=u'ā˜ƒ', those=4) - ea.current_es.search.return_value = hits + hits = generate_hits([START_TIMESTAMP, END_TIMESTAMP], this='abc', that='ā˜ƒ', those=4) + ea.thread_data.current_es.search.return_value = hits ea.run_query(ea.rules[0], START, END) call_args = ea.rules[0]['type'].add_data.call_args_list[0] assert 'this,that,those' in call_args[0][0][0] - assert call_args[0][0][0]['this,that,those'] == u'abc, ā˜ƒ, 4' + assert call_args[0][0][0]['this,that,those'] == 'abc, ā˜ƒ, 4' def test_silence_query_key(ea): @@ -604,7 +671,7 @@ def test_silence_query_key(ea): def test_realert(ea): hits = ['2014-09-26T12:35:%sZ' % (x) for x in range(60)] matches = [{'@timestamp': x} for x in hits] - ea.current_es.search.return_value = hits + ea.thread_data.current_es.search.return_value = hits with mock.patch('elastalert.elastalert.elasticsearch_client'): ea.rules[0]['realert'] = datetime.timedelta(seconds=50) ea.rules[0]['type'].matches = matches @@ -691,19 +758,21 @@ def test_realert_with_nested_query_key(ea): def test_count(ea): ea.rules[0]['use_count_query'] = True ea.rules[0]['doc_type'] = 'doctype' - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client'), \ + mock.patch.object(ea, 'get_hits_count') as mock_hits: ea.run_rule(ea.rules[0], END, START) # Assert that es.count is run against every run_every timeframe between START and END start = START query = { - 'query': {'filtered': {'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}} + 'query': {'filtered': { + 'filter': {'bool': {'must': [{'range': {'@timestamp': {'lte': END_TIMESTAMP, 'gt': START_TIMESTAMP}}}]}}}}} while END - start > ea.run_every: end = start + ea.run_every query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['lte'] = dt_to_ts(end) query['query']['filtered']['filter']['bool']['must'][0]['range']['@timestamp']['gt'] = dt_to_ts(start) + mock_hits.assert_any_call(mock.ANY, start, end, mock.ANY) start = start + ea.run_every - ea.current_es.count.assert_any_call(body=query, doc_type='doctype', index='idx', ignore_unavailable=True) def run_and_assert_segmented_queries(ea, start, end, segment_size): @@ -718,7 +787,8 @@ def run_and_assert_segmented_queries(ea, start, end, segment_size): # Assert elastalert_status was created for the entire time range assert ea.writeback_es.index.call_args_list[-1][1]['body']['starttime'] == dt_to_ts(original_start) if ea.rules[0].get('aggregation_query_element'): - assert ea.writeback_es.index.call_args_list[-1][1]['body']['endtime'] == dt_to_ts(original_end - (original_end - end)) + assert ea.writeback_es.index.call_args_list[-1][1]['body']['endtime'] == dt_to_ts( + original_end - (original_end - end)) assert original_end - end < segment_size else: assert ea.writeback_es.index.call_args_list[-1][1]['body']['endtime'] == dt_to_ts(original_end) @@ -727,8 +797,8 @@ def run_and_assert_segmented_queries(ea, start, end, segment_size): def test_query_segmenting_reset_num_hits(ea): # Tests that num_hits gets reset every time run_query is run def assert_num_hits_reset(): - assert ea.num_hits == 0 - ea.num_hits += 10 + assert ea.thread_data.num_hits == 0 + ea.thread_data.num_hits += 10 with mock.patch.object(ea, 'run_query') as mock_run_query: mock_run_query.side_effect = assert_num_hits_reset() ea.run_rule(ea.rules[0], END, START) @@ -845,7 +915,8 @@ def test_set_starttime(ea): assert ea.rules[0]['starttime'] == end - ea.run_every # Count query, with previous endtime - with mock.patch('elastalert.elastalert.elasticsearch_client'): + with mock.patch('elastalert.elastalert.elasticsearch_client'), \ + mock.patch.object(ea, 'get_hits_count'): ea.run_rule(ea.rules[0], END, START) ea.set_starttime(ea.rules[0], end) assert ea.rules[0]['starttime'] == END @@ -856,6 +927,12 @@ def test_set_starttime(ea): ea.set_starttime(ea.rules[0], end) assert ea.rules[0]['starttime'] == ea.rules[0]['previous_endtime'] + # Make sure starttime is updated if previous_endtime isn't used + ea.rules[0]['previous_endtime'] = end - ea.buffer_time / 2 + ea.rules[0]['starttime'] = ts_to_dt('2014-10-09T00:00:01') + ea.set_starttime(ea.rules[0], end) + assert ea.rules[0]['starttime'] == end - ea.buffer_time + # scan_entire_timeframe ea.rules[0].pop('previous_endtime') ea.rules[0].pop('starttime') @@ -875,15 +952,15 @@ def test_kibana_dashboard(ea): mock_es_init.return_value = mock_es # No dashboard found - mock_es.search.return_value = {'hits': {'total': 0, 'hits': []}} + mock_es.deprecated_search.return_value = {'hits': {'total': 0, 'hits': []}} with pytest.raises(EAException): ea.use_kibana_link(ea.rules[0], match) - mock_call = mock_es.search.call_args_list[0][1] + mock_call = mock_es.deprecated_search.call_args_list[0][1] assert mock_call['body'] == {'query': {'term': {'_id': 'my dashboard'}}} # Dashboard found mock_es.index.return_value = {'_id': 'ABCDEFG'} - mock_es.search.return_value = {'hits': {'hits': [{'_source': {'dashboard': json.dumps(dashboard_temp)}}]}} + mock_es.deprecated_search.return_value = {'hits': {'hits': [{'_source': {'dashboard': json.dumps(dashboard_temp)}}]}} url = ea.use_kibana_link(ea.rules[0], match) assert 'ABCDEFG' in url db = json.loads(mock_es.index.call_args_list[0][1]['body']['dashboard']) @@ -906,9 +983,9 @@ def test_kibana_dashboard(ea): url = ea.use_kibana_link(ea.rules[0], match) db = json.loads(mock_es.index.call_args_list[-1][1]['body']['dashboard']) found_filters = 0 - for filter_id, filter_dict in db['services']['filter']['list'].items(): + for filter_id, filter_dict in list(db['services']['filter']['list'].items()): if (filter_dict['field'] == 'foo' and filter_dict['query'] == '"cat"') or \ - (filter_dict['field'] == 'bar' and filter_dict['query'] == '"dog"'): + (filter_dict['field'] == 'bar' and filter_dict['query'] == '"dog"'): found_filters += 1 continue assert found_filters == 2 @@ -917,17 +994,20 @@ def test_kibana_dashboard(ea): def test_rule_changes(ea): ea.rule_hashes = {'rules/rule1.yaml': 'ABC', 'rules/rule2.yaml': 'DEF'} - ea.rules = [ea.init_rule(rule, True) for rule in [{'rule_file': 'rules/rule1.yaml', 'name': 'rule1', 'filter': []}, - {'rule_file': 'rules/rule2.yaml', 'name': 'rule2', 'filter': []}]] + run_every = datetime.timedelta(seconds=1) + ea.rules = [ea.init_rule(rule, True) for rule in [{'rule_file': 'rules/rule1.yaml', 'name': 'rule1', 'filter': [], + 'run_every': run_every}, + {'rule_file': 'rules/rule2.yaml', 'name': 'rule2', 'filter': [], + 'run_every': run_every}]] ea.rules[1]['processed_hits'] = ['save me'] new_hashes = {'rules/rule1.yaml': 'ABC', 'rules/rule3.yaml': 'XXX', 'rules/rule2.yaml': '!@#$'} - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: - mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml'}, - {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml'}] + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: + mock_load.side_effect = [{'filter': [], 'name': 'rule2', 'rule_file': 'rules/rule2.yaml', 'run_every': run_every}, + {'filter': [], 'name': 'rule3', 'rule_file': 'rules/rule3.yaml', 'run_every': run_every}] mock_hashes.return_value = new_hashes ea.load_rule_changes() @@ -945,10 +1025,11 @@ def test_rule_changes(ea): # A new rule with a conflicting name wont load new_hashes = copy.copy(new_hashes) new_hashes.update({'rules/rule4.yaml': 'asdf'}) - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: with mock.patch.object(ea, 'send_notification_email') as mock_send: - mock_load.return_value = {'filter': [], 'name': 'rule3', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} + mock_load.return_value = {'filter': [], 'name': 'rule3', 'new': 'stuff', + 'rule_file': 'rules/rule4.yaml', 'run_every': run_every} mock_hashes.return_value = new_hashes ea.load_rule_changes() mock_send.assert_called_once_with(exception=mock.ANY, rule_file='rules/rule4.yaml') @@ -958,9 +1039,10 @@ def test_rule_changes(ea): # A new rule with is_enabled=False wont load new_hashes = copy.copy(new_hashes) new_hashes.update({'rules/rule4.yaml': 'asdf'}) - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: - mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, 'rule_file': 'rules/rule4.yaml'} + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'is_enabled': False, + 'rule_file': 'rules/rule4.yaml', 'run_every': run_every} mock_hashes.return_value = new_hashes ea.load_rule_changes() assert len(ea.rules) == 3 @@ -969,13 +1051,24 @@ def test_rule_changes(ea): # An old rule which didn't load gets reloaded new_hashes = copy.copy(new_hashes) new_hashes['rules/rule4.yaml'] = 'qwerty' - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: - mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml'} + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml', + 'run_every': run_every} mock_hashes.return_value = new_hashes ea.load_rule_changes() assert len(ea.rules) == 4 + # Disable a rule by removing the file + new_hashes.pop('rules/rule4.yaml') + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: + mock_load.return_value = {'filter': [], 'name': 'rule4', 'new': 'stuff', 'rule_file': 'rules/rule4.yaml', + 'run_every': run_every} + mock_hashes.return_value = new_hashes + ea.load_rule_changes() + ea.scheduler.remove_job.assert_called_with(job_id='rule4') + def test_strf_index(ea): """ Test that the get_index function properly generates indexes spanning days """ @@ -1002,13 +1095,16 @@ def test_count_keys(ea): ea.rules[0]['top_count_keys'] = ['this', 'that'] ea.rules[0]['type'].matches = {'@timestamp': END} ea.rules[0]['doc_type'] = 'blah' - buckets = [{'aggregations': {'filtered': {'counts': {'buckets': [{'key': 'a', 'doc_count': 10}, {'key': 'b', 'doc_count': 5}]}}}}, - {'aggregations': {'filtered': {'counts': {'buckets': [{'key': 'd', 'doc_count': 10}, {'key': 'c', 'doc_count': 12}]}}}}] - ea.current_es.search.side_effect = buckets + buckets = [{'aggregations': { + 'filtered': {'counts': {'buckets': [{'key': 'a', 'doc_count': 10}, {'key': 'b', 'doc_count': 5}]}}}}, + {'aggregations': {'filtered': { + 'counts': {'buckets': [{'key': 'd', 'doc_count': 10}, {'key': 'c', 'doc_count': 12}]}}}}] + ea.thread_data.current_es.deprecated_search.side_effect = buckets counts = ea.get_top_counts(ea.rules[0], START, END, ['this', 'that']) - calls = ea.current_es.search.call_args_list + calls = ea.thread_data.current_es.deprecated_search.call_args_list assert calls[0][1]['search_type'] == 'count' - assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5} + assert calls[0][1]['body']['aggs']['filtered']['aggs']['counts']['terms'] == {'field': 'this', 'size': 5, + 'min_doc_count': 1} assert counts['top_events_this'] == {'a': 10, 'b': 5} assert counts['top_events_that'] == {'d': 10, 'c': 12} @@ -1024,13 +1120,13 @@ def test_exponential_realert(ea): ts5m = until + datetime.timedelta(minutes=5) ts4h = until + datetime.timedelta(hours=4) - test_values = [(ts5s, until, 0), # Exp will increase to 1, 10*2**0 = 10s + test_values = [(ts5s, until, 0), # Exp will increase to 1, 10*2**0 = 10s (ts15s, until, 0), # Exp will stay at 0, 10*2**0 = 10s (ts15s, until, 1), # Exp will increase to 2, 10*2**1 = 20s - (ts1m, until, 2), # Exp will decrease to 1, 10*2**2 = 40s - (ts1m, until, 3), # Exp will increase to 4, 10*2**3 = 1m20s - (ts5m, until, 1), # Exp will lower back to 0, 10*2**1 = 20s - (ts4h, until, 9), # Exp will lower back to 0, 10*2**9 = 1h25m + (ts1m, until, 2), # Exp will decrease to 1, 10*2**2 = 40s + (ts1m, until, 3), # Exp will increase to 4, 10*2**3 = 1m20s + (ts5m, until, 1), # Exp will lower back to 0, 10*2**1 = 20s + (ts4h, until, 9), # Exp will lower back to 0, 10*2**9 = 1h25m (ts4h, until, 10), # Exp will lower back to 9, 10*2**10 = 2h50m (ts4h, until, 11)] # Exp will increase to 12, 10*2**11 = 5h results = (1, 0, 2, 1, 4, 0, 0, 9, 12) @@ -1038,7 +1134,7 @@ def test_exponential_realert(ea): for args in test_values: ea.silence_cache[ea.rules[0]['name']] = (args[1], args[2]) next_alert, exponent = ea.next_alert_time(ea.rules[0], ea.rules[0]['name'], args[0]) - assert exponent == next_res.next() + assert exponent == next(next_res) def test_wait_until_responsive(ea): @@ -1047,7 +1143,7 @@ def test_wait_until_responsive(ea): # Takes a while before becoming responsive. ea.writeback_es.indices.exists.side_effect = [ ConnectionError(), # ES is not yet responsive. - False, # index does not yet exist. + False, # index does not yet exist. True, ] @@ -1107,7 +1203,7 @@ def test_wait_until_responsive_timeout_index_does_not_exist(ea, capsys): # Ensure we get useful diagnostics. output, errors = capsys.readouterr() - assert 'Writeback index "wb" does not exist, did you run `elastalert-create-index`?' in errors + assert 'Writeback alias "wb_a" does not exist, did you run `elastalert-create-index`?' in errors # Slept until we passed the deadline. sleep.mock_calls == [ @@ -1131,7 +1227,7 @@ def mock_loop(): ea.stop() with mock.patch.object(ea, 'sleep_for', return_value=None): - with mock.patch.object(ea, 'run_all_rules') as mock_run: + with mock.patch.object(ea, 'sleep_for') as mock_run: mock_run.side_effect = mock_loop() start_thread = threading.Thread(target=ea.start) # Set as daemon to prevent a failed test from blocking exit @@ -1190,8 +1286,8 @@ def test_uncaught_exceptions(ea): # Changing the file should re-enable it ea.rule_hashes = {'blah.yaml': 'abc'} new_hashes = {'blah.yaml': 'def'} - with mock.patch('elastalert.elastalert.get_rule_hashes') as mock_hashes: - with mock.patch('elastalert.elastalert.load_configuration') as mock_load: + with mock.patch.object(ea.conf['rules_loader'], 'get_hashes') as mock_hashes: + with mock.patch.object(ea.conf['rules_loader'], 'load_configuration') as mock_load: mock_load.side_effect = [ea.disabled_rules[0]] mock_hashes.return_value = new_hashes ea.load_rule_changes() @@ -1250,15 +1346,15 @@ def test_query_with_whitelist_filter_es(ea): in new_rule['filter'][-1]['query']['query_string']['query'] -def test_query_with_whitelist_filter_es_five(ea): - ea.es_version = '6.2' - ea.rules[0]['_source_enabled'] = False - ea.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}] - ea.rules[0]['compare_key'] = "username" - ea.rules[0]['whitelist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] - new_rule = copy.copy(ea.rules[0]) - ea.init_rule(new_rule, True) - assert 'NOT username:"xudan1" AND NOT username:"xudan12" AND NOT username:"aa1"' in new_rule['filter'][-1]['query_string']['query'] +def test_query_with_whitelist_filter_es_five(ea_sixsix): + ea_sixsix.rules[0]['_source_enabled'] = False + ea_sixsix.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}] + ea_sixsix.rules[0]['compare_key'] = "username" + ea_sixsix.rules[0]['whitelist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] + new_rule = copy.copy(ea_sixsix.rules[0]) + ea_sixsix.init_rule(new_rule, True) + assert 'NOT username:"xudan1" AND NOT username:"xudan12" AND NOT username:"aa1"' in \ + new_rule['filter'][-1]['query_string']['query'] def test_query_with_blacklist_filter_es(ea): @@ -1268,15 +1364,17 @@ def test_query_with_blacklist_filter_es(ea): ea.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] new_rule = copy.copy(ea.rules[0]) ea.init_rule(new_rule, True) - assert 'username:"xudan1" OR username:"xudan12" OR username:"aa1"' in new_rule['filter'][-1]['query']['query_string']['query'] - - -def test_query_with_blacklist_filter_es_five(ea): - ea.es_version = '6.2' - ea.rules[0]['_source_enabled'] = False - ea.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}] - ea.rules[0]['compare_key'] = "username" - ea.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] - new_rule = copy.copy(ea.rules[0]) - ea.init_rule(new_rule, True) - assert 'username:"xudan1" OR username:"xudan12" OR username:"aa1"' in new_rule['filter'][-1]['query_string']['query'] + assert 'username:"xudan1" OR username:"xudan12" OR username:"aa1"' in \ + new_rule['filter'][-1]['query']['query_string']['query'] + + +def test_query_with_blacklist_filter_es_five(ea_sixsix): + ea_sixsix.rules[0]['_source_enabled'] = False + ea_sixsix.rules[0]['filter'] = [{'query_string': {'query': 'baz'}}] + ea_sixsix.rules[0]['compare_key'] = "username" + ea_sixsix.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] + ea_sixsix.rules[0]['blacklist'] = ['xudan1', 'xudan12', 'aa1', 'bb1'] + new_rule = copy.copy(ea_sixsix.rules[0]) + ea_sixsix.init_rule(new_rule, True) + assert 'username:"xudan1" OR username:"xudan12" OR username:"aa1"' in new_rule['filter'][-1]['query_string'][ + 'query'] diff --git a/tests/conftest.py b/tests/conftest.py index ca50a101a..6844296ee 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,9 +1,9 @@ # -*- coding: utf-8 -*- import datetime - import logging -import mock import os + +import mock import pytest import elastalert.elastalert @@ -11,8 +11,28 @@ from elastalert.util import dt_to_ts from elastalert.util import ts_to_dt +writeback_index = 'wb' + + +def pytest_addoption(parser): + parser.addoption( + "--runelasticsearch", action="store_true", default=False, help="run elasticsearch tests" + ) + -mock_info = {'status': 200, 'name': 'foo', 'version': {'number': '2.0'}} +def pytest_collection_modifyitems(config, items): + if config.getoption("--runelasticsearch"): + # --runelasticsearch given in cli: run elasticsearch tests, skip ordinary unit tests + skip_unit_tests = pytest.mark.skip(reason="not running when --runelasticsearch option is used to run") + for item in items: + if "elasticsearch" not in item.keywords: + item.add_marker(skip_unit_tests) + else: + # skip elasticsearch tests + skip_elasticsearch = pytest.mark.skip(reason="need --runelasticsearch option to run") + for item in items: + if "elasticsearch" in item.keywords: + item.add_marker(skip_elasticsearch) @pytest.fixture(scope='function', autouse=True) @@ -43,12 +63,62 @@ def __init__(self, host='es', port=14900): self.port = port self.return_hits = [] self.search = mock.Mock() + self.deprecated_search = mock.Mock() + self.create = mock.Mock() + self.index = mock.Mock() + self.delete = mock.Mock() + self.info = mock.Mock(return_value={'status': 200, 'name': 'foo', 'version': {'number': '2.0'}}) + self.ping = mock.Mock(return_value=True) + self.indices = mock_es_indices_client() + self.es_version = mock.Mock(return_value='2.0') + self.is_atleastfive = mock.Mock(return_value=False) + self.is_atleastsix = mock.Mock(return_value=False) + self.is_atleastsixtwo = mock.Mock(return_value=False) + self.is_atleastsixsix = mock.Mock(return_value=False) + self.is_atleastseven = mock.Mock(return_value=False) + self.resolve_writeback_index = mock.Mock(return_value=writeback_index) + + +class mock_es_sixsix_client(object): + def __init__(self, host='es', port=14900): + self.host = host + self.port = port + self.return_hits = [] + self.search = mock.Mock() + self.deprecated_search = mock.Mock() self.create = mock.Mock() self.index = mock.Mock() self.delete = mock.Mock() - self.info = mock.Mock(return_value=mock_info) + self.info = mock.Mock(return_value={'status': 200, 'name': 'foo', 'version': {'number': '6.6.0'}}) self.ping = mock.Mock(return_value=True) self.indices = mock_es_indices_client() + self.es_version = mock.Mock(return_value='6.6.0') + self.is_atleastfive = mock.Mock(return_value=True) + self.is_atleastsix = mock.Mock(return_value=True) + self.is_atleastsixtwo = mock.Mock(return_value=False) + self.is_atleastsixsix = mock.Mock(return_value=True) + self.is_atleastseven = mock.Mock(return_value=False) + + def writeback_index_side_effect(index, doc_type): + if doc_type == 'silence': + return index + '_silence' + elif doc_type == 'past_elastalert': + return index + '_past' + elif doc_type == 'elastalert_status': + return index + '_status' + elif doc_type == 'elastalert_error': + return index + '_error' + return index + + self.resolve_writeback_index = mock.Mock(side_effect=writeback_index_side_effect) + + +class mock_rule_loader(object): + def __init__(self, conf): + self.base_config = conf + self.load = mock.Mock() + self.get_hashes = mock.Mock() + self.load_configuration = mock.Mock() class mock_ruletype(object): @@ -87,7 +157,8 @@ def ea(): 'max_query_size': 10000, 'ts_to_dt': ts_to_dt, 'dt_to_ts': dt_to_ts, - '_source_enabled': True}] + '_source_enabled': True, + 'run_every': datetime.timedelta(seconds=15)}] conf = {'rules_folder': 'rules', 'run_every': datetime.timedelta(minutes=10), 'buffer_time': datetime.timedelta(minutes=5), @@ -95,30 +166,90 @@ def ea(): 'es_host': 'es', 'es_port': 14900, 'writeback_index': 'wb', + 'writeback_alias': 'wb_a', 'rules': rules, 'max_query_size': 10000, 'old_query_limit': datetime.timedelta(weeks=1), 'disable_rules_on_error': False, 'scroll_keepalive': '30s'} + elastalert.util.elasticsearch_client = mock_es_client + conf['rules_loader'] = mock_rule_loader(conf) elastalert.elastalert.elasticsearch_client = mock_es_client - with mock.patch('elastalert.elastalert.get_rule_hashes'): - with mock.patch('elastalert.elastalert.load_rules') as load_conf: + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + with mock.patch('elastalert.elastalert.BackgroundScheduler'): load_conf.return_value = conf + conf['rules_loader'].load.return_value = rules + conf['rules_loader'].get_hashes.return_value = {} ea = elastalert.elastalert.ElastAlerter(['--pin_rules']) ea.rules[0]['type'] = mock_ruletype() ea.rules[0]['alert'] = [mock_alert()] ea.writeback_es = mock_es_client() - ea.writeback_es.search.return_value = {'hits': {'hits': []}} - ea.writeback_es.index.return_value = {'_id': 'ABCD'} + ea.writeback_es.search.return_value = {'hits': {'hits': []}, 'total': 0} + ea.writeback_es.deprecated_search.return_value = {'hits': {'hits': []}} + ea.writeback_es.index.return_value = {'_id': 'ABCD', 'created': True} ea.current_es = mock_es_client('', '') + ea.thread_data.current_es = ea.current_es + ea.thread_data.num_hits = 0 + ea.thread_data.num_dupes = 0 return ea +@pytest.fixture +def ea_sixsix(): + rules = [{'es_host': '', + 'es_port': 14900, + 'name': 'anytest', + 'index': 'idx', + 'filter': [], + 'include': ['@timestamp'], + 'run_every': datetime.timedelta(seconds=1), + 'aggregation': datetime.timedelta(0), + 'realert': datetime.timedelta(0), + 'processed_hits': {}, + 'timestamp_field': '@timestamp', + 'match_enhancements': [], + 'rule_file': 'blah.yaml', + 'max_query_size': 10000, + 'ts_to_dt': ts_to_dt, + 'dt_to_ts': dt_to_ts, + '_source_enabled': True}] + conf = {'rules_folder': 'rules', + 'run_every': datetime.timedelta(minutes=10), + 'buffer_time': datetime.timedelta(minutes=5), + 'alert_time_limit': datetime.timedelta(hours=24), + 'es_host': 'es', + 'es_port': 14900, + 'writeback_index': writeback_index, + 'writeback_alias': 'wb_a', + 'rules': rules, + 'max_query_size': 10000, + 'old_query_limit': datetime.timedelta(weeks=1), + 'disable_rules_on_error': False, + 'scroll_keepalive': '30s'} + conf['rules_loader'] = mock_rule_loader(conf) + elastalert.elastalert.elasticsearch_client = mock_es_sixsix_client + elastalert.util.elasticsearch_client = mock_es_sixsix_client + with mock.patch('elastalert.elastalert.load_conf') as load_conf: + with mock.patch('elastalert.elastalert.BackgroundScheduler'): + load_conf.return_value = conf + conf['rules_loader'].load.return_value = rules + conf['rules_loader'].get_hashes.return_value = {} + ea_sixsix = elastalert.elastalert.ElastAlerter(['--pin_rules']) + ea_sixsix.rules[0]['type'] = mock_ruletype() + ea_sixsix.rules[0]['alert'] = [mock_alert()] + ea_sixsix.writeback_es = mock_es_sixsix_client() + ea_sixsix.writeback_es.search.return_value = {'hits': {'hits': []}} + ea_sixsix.writeback_es.deprecated_search.return_value = {'hits': {'hits': []}} + ea_sixsix.writeback_es.index.return_value = {'_id': 'ABCD'} + ea_sixsix.current_es = mock_es_sixsix_client('', -1) + return ea_sixsix + + @pytest.fixture(scope='function') def environ(): """py.test fixture to get a fresh mutable environment.""" old_env = os.environ - new_env = dict(old_env.items()) + new_env = dict(list(old_env.items())) os.environ = new_env yield os.environ os.environ = old_env diff --git a/tests/create_index_test.py b/tests/create_index_test.py new file mode 100644 index 000000000..47a6247dc --- /dev/null +++ b/tests/create_index_test.py @@ -0,0 +1,53 @@ +# -*- coding: utf-8 -*- +import json + +import pytest + +import elastalert.create_index + +es_mappings = [ + 'elastalert', + 'elastalert_error', + 'elastalert_status', + 'past_elastalert', + 'silence' +] + + +@pytest.mark.parametrize('es_mapping', es_mappings) +def test_read_default_index_mapping(es_mapping): + mapping = elastalert.create_index.read_es_index_mapping(es_mapping) + assert es_mapping not in mapping + print((json.dumps(mapping, indent=2))) + + +@pytest.mark.parametrize('es_mapping', es_mappings) +def test_read_es_5_index_mapping(es_mapping): + mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 5) + assert es_mapping in mapping + print((json.dumps(mapping, indent=2))) + + +@pytest.mark.parametrize('es_mapping', es_mappings) +def test_read_es_6_index_mapping(es_mapping): + mapping = elastalert.create_index.read_es_index_mapping(es_mapping, 6) + assert es_mapping not in mapping + print((json.dumps(mapping, indent=2))) + + +def test_read_default_index_mappings(): + mappings = elastalert.create_index.read_es_index_mappings() + assert len(mappings) == len(es_mappings) + print((json.dumps(mappings, indent=2))) + + +def test_read_es_5_index_mappings(): + mappings = elastalert.create_index.read_es_index_mappings(5) + assert len(mappings) == len(es_mappings) + print((json.dumps(mappings, indent=2))) + + +def test_read_es_6_index_mappings(): + mappings = elastalert.create_index.read_es_index_mappings(6) + assert len(mappings) == len(es_mappings) + print((json.dumps(mappings, indent=2))) diff --git a/tests/elasticsearch_test.py b/tests/elasticsearch_test.py new file mode 100644 index 000000000..308356c25 --- /dev/null +++ b/tests/elasticsearch_test.py @@ -0,0 +1,102 @@ +# -*- coding: utf-8 -*- +import datetime +import json +import time + +import dateutil +import pytest + +import elastalert.create_index +import elastalert.elastalert +from elastalert import ElasticSearchClient +from elastalert.util import build_es_conn_config +from tests.conftest import ea # noqa: F401 + +test_index = 'test_index' + +es_host = '127.0.0.1' +es_port = 9200 +es_timeout = 10 + + +@pytest.fixture +def es_client(): + es_conn_config = build_es_conn_config({'es_host': es_host, 'es_port': es_port, 'es_conn_timeout': es_timeout}) + return ElasticSearchClient(es_conn_config) + + +@pytest.mark.elasticsearch +class TestElasticsearch(object): + # TODO perform teardown removing data inserted into Elasticsearch + # Warning!!!: Test class is not erasing its testdata on the Elasticsearch server. + # This is not a problem as long as the data is manually removed or the test environment + # is torn down after the test run(eg. running tests in a test environment such as Travis) + def test_create_indices(self, es_client): + elastalert.create_index.create_index_mappings(es_client=es_client, ea_index=test_index) + indices_mappings = es_client.indices.get_mapping(test_index + '*') + print(('-' * 50)) + print((json.dumps(indices_mappings, indent=2))) + print(('-' * 50)) + if es_client.is_atleastsix(): + assert test_index in indices_mappings + assert test_index + '_error' in indices_mappings + assert test_index + '_status' in indices_mappings + assert test_index + '_silence' in indices_mappings + assert test_index + '_past' in indices_mappings + else: + assert 'elastalert' in indices_mappings[test_index]['mappings'] + assert 'elastalert_error' in indices_mappings[test_index]['mappings'] + assert 'elastalert_status' in indices_mappings[test_index]['mappings'] + assert 'silence' in indices_mappings[test_index]['mappings'] + assert 'past_elastalert' in indices_mappings[test_index]['mappings'] + + @pytest.mark.usefixtures("ea") + def test_aggregated_alert(self, ea, es_client): # noqa: F811 + match_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( + days=1) + ea.rules[0]['aggregate_by_match_time'] = True + match = {'@timestamp': match_timestamp, + 'num_hits': 0, + 'num_matches': 3 + } + ea.writeback_es = es_client + res = ea.add_aggregated_alert(match, ea.rules[0]) + if ea.writeback_es.is_atleastsix(): + assert res['result'] == 'created' + else: + assert res['created'] is True + # Make sure added data is available for querying + time.sleep(2) + # Now lets find the pending aggregated alert + assert ea.find_pending_aggregate_alert(ea.rules[0]) + + @pytest.mark.usefixtures("ea") + def test_silenced(self, ea, es_client): # noqa: F811 + until_timestamp = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + datetime.timedelta( + days=1) + ea.writeback_es = es_client + res = ea.set_realert(ea.rules[0]['name'], until_timestamp, 0) + if ea.writeback_es.is_atleastsix(): + assert res['result'] == 'created' + else: + assert res['created'] is True + # Make sure added data is available for querying + time.sleep(2) + # Force lookup in elasticsearch + ea.silence_cache = {} + # Now lets check if our rule is reported as silenced + assert ea.is_silenced(ea.rules[0]['name']) + + @pytest.mark.usefixtures("ea") + def test_get_hits(self, ea, es_client): # noqa: F811 + start = datetime.datetime.now(tz=dateutil.tz.tzutc()).replace(microsecond=0) + end = start + datetime.timedelta(days=1) + ea.current_es = es_client + if ea.current_es.is_atleastfive(): + ea.rules[0]['five'] = True + else: + ea.rules[0]['five'] = False + ea.thread_data.current_es = ea.current_es + hits = ea.get_hits(ea.rules[0], start, end, test_index) + + assert isinstance(hits, list) diff --git a/tests/kibana_discover_test.py b/tests/kibana_discover_test.py new file mode 100644 index 000000000..f06fe4e0c --- /dev/null +++ b/tests/kibana_discover_test.py @@ -0,0 +1,858 @@ +# -*- coding: utf-8 -*- +from datetime import timedelta +import pytest + +from elastalert.kibana_discover import generate_kibana_discover_url + + +@pytest.mark.parametrize("kibana_version", ['5.6', '6.0', '6.1', '6.2', '6.3', '6.4', '6.5', '6.6', '6.7', '6.8']) +def test_generate_kibana_discover_url_with_kibana_5x_and_6x(kibana_version): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': kibana_version, + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +@pytest.mark.parametrize("kibana_version", ['7.0', '7.1', '7.2', '7.3']) +def test_generate_kibana_discover_url_with_kibana_7x(kibana_version): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': kibana_version, + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_missing_kibana_discover_version(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_kibana_discover_url_with_missing_kibana_discover_app_url(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_kibana_discover_url_with_missing_kibana_discover_index_pattern_id(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'timestamp_field': 'timestamp', + 'name': 'test' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_kibana_discover_url_with_invalid_kibana_version(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '4.5', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + assert url is None + + +def test_generate_kibana_discover_url_with_kibana_discover_app_url_env_substitution(environ): + environ.update({ + 'KIBANA_HOST': 'kibana', + 'KIBANA_PORT': '5601', + }) + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://$KIBANA_HOST:$KIBANA_PORT/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_from_timedelta(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_from_timedelta': timedelta(hours=1), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A10%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_from_timedelta_and_timeframe(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_from_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A00%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A20%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_to_timedelta(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_to_timedelta': timedelta(hours=1), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A50%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_to_timedelta_and_timeframe(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'kibana_discover_to_timedelta': timedelta(hours=1), + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:00:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T03%3A40%3A00Z%27%2C' + + 'to%3A%272019-09-01T05%3A00%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_timeframe(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '7.3', + 'kibana_discover_index_pattern_id': 'd6cabfb6-aaef-44ea-89c5-600e9a76991a', + 'timeframe': timedelta(minutes=20), + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T04:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'filters%3A%21%28%29%2C' + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T04%3A10%3A00Z%27%2C' + + 'to%3A%272019-09-01T04%3A50%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3Ad6cabfb6-aaef-44ea-89c5-600e9a76991a%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_custom_columns(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'kibana_discover_columns': ['level', 'message'], + 'timestamp_field': 'timestamp' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28level%2Cmessage%29%2C' + + 'filters%3A%21%28%29%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_single_filter(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_multiple_filters(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': '90943e30-9a47-11e8-b64d-95841ca0b247', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'app': 'test'}}, + {'term': {'level': 30}} + ] + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28app%3Atest%29%29%2C%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B' # value start + + '%7B%22term%22%3A%7B%22app%22%3A%22test%22%7D%7D%2C%7B%22term%22%3A%7B%22level%22%3A30%7D%7D' + + '%5D%7D%27' # value end + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%2790943e30-9a47-11e8-b64d-95841ca0b247%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_int_query_key(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo.dest': 200 + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3A200%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3A%27200%27' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # reponse start + + 'query%3A200%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_str_query_key(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'dest': 'ok' + } + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3Aok%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3Aok' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # geo.dest start + + 'query%3Aok%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_null_query_key_value(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': None + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'exists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21t%2C' + + 'type%3Aexists%2C' + + 'value%3Aexists' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_missing_query_key_value(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'exists%3A%28field%3Astatus%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21t%2C' + + 'type%3Aexists%2C' + + 'value%3Aexists' + + '%29' # meta end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_compound_query_key(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'compound_query_key': ['geo.src', 'geo.dest'], + 'query_key': 'geo.src,geo.dest' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'geo': { + 'src': 'CA', + 'dest': 'US' + } + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # geo.src filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.src%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3ACA%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3ACA' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.src%3A%28' # reponse start + + 'query%3ACA%2C' + + 'type%3Aphrase' + + '%29' # geo.src end + + '%29' # match end + + '%29' # query end + + '%29%2C' # geo.src filter end + + + '%28' # geo.dest filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Ageo.dest%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3AUS%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3AUS' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'geo.dest%3A%28' # geo.dest start + + 'query%3AUS%2C' + + 'type%3Aphrase' + + '%29' # geo.dest end + + '%29' # match end + + '%29' # query end + + '%29' # geo.dest filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl + + +def test_generate_kibana_discover_url_with_filter_and_query_key(): + url = generate_kibana_discover_url( + rule={ + 'kibana_discover_app_url': 'http://kibana:5601/#/discover', + 'kibana_discover_version': '6.8', + 'kibana_discover_index_pattern_id': 'logs-*', + 'timestamp_field': 'timestamp', + 'filter': [ + {'term': {'level': 30}} + ], + 'query_key': 'status' + }, + match={ + 'timestamp': '2019-09-01T00:30:00Z', + 'status': 'ok' + } + ) + expectedUrl = ( + 'http://kibana:5601/#/discover' + + '?_g=%28' # global start + + 'refreshInterval%3A%28pause%3A%21t%2Cvalue%3A0%29%2C' + + 'time%3A%28' # time start + + 'from%3A%272019-09-01T00%3A20%3A00Z%27%2C' + + 'mode%3Aabsolute%2C' + + 'to%3A%272019-09-01T00%3A40%3A00Z%27' + + '%29' # time end + + '%29' # global end + + '&_a=%28' # app start + + 'columns%3A%21%28_source%29%2C' + + 'filters%3A%21%28' # filters start + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'bool%3A%28must%3A%21%28%28term%3A%28level%3A30%29%29%29%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3Afilter%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Abool%2C' + + 'negate%3A%21f%2C' + + 'type%3Acustom%2C' + + 'value%3A%27%7B%22must%22%3A%5B%7B%22term%22%3A%7B%22level%22%3A30%7D%7D%5D%7D%27' + + '%29' # meta end + + '%29%2C' # filter end + + + '%28' # filter start + + '%27%24state%27%3A%28store%3AappState%29%2C' + + 'meta%3A%28' # meta start + + 'alias%3A%21n%2C' + + 'disabled%3A%21f%2C' + + 'index%3A%27logs-%2A%27%2C' + + 'key%3Astatus%2C' + + 'negate%3A%21f%2C' + + 'params%3A%28query%3Aok%2C' # params start + + 'type%3Aphrase' + + '%29%2C' # params end + + 'type%3Aphrase%2C' + + 'value%3Aok' + + '%29%2C' # meta end + + 'query%3A%28' # query start + + 'match%3A%28' # match start + + 'status%3A%28' # status start + + 'query%3Aok%2C' + + 'type%3Aphrase' + + '%29' # status end + + '%29' # match end + + '%29' # query end + + '%29' # filter end + + + '%29%2C' # filters end + + 'index%3A%27logs-%2A%27%2C' + + 'interval%3Aauto' + + '%29' # app end + ) + assert url == expectedUrl diff --git a/tests/config_test.py b/tests/loaders_test.py similarity index 50% rename from tests/config_test.py rename to tests/loaders_test.py index f444f0e25..bb8d3d873 100644 --- a/tests/config_test.py +++ b/tests/loaders_test.py @@ -1,27 +1,24 @@ # -*- coding: utf-8 -*- import copy import datetime +import os import mock -import os import pytest import elastalert.alerts import elastalert.ruletypes -from elastalert.config import get_file_paths -from elastalert.config import load_configuration -from elastalert.config import load_modules -from elastalert.config import load_options -from elastalert.config import load_rules +from elastalert.config import load_conf +from elastalert.loaders import FileRulesLoader from elastalert.util import EAException -from elastalert.config import import_rules test_config = {'rules_folder': 'test_folder', 'run_every': {'minutes': 10}, 'buffer_time': {'minutes': 10}, 'es_host': 'elasticsearch.test', 'es_port': 12345, - 'writeback_index': 'test_index'} + 'writeback_index': 'test_index', + 'writeback_alias': 'test_alias'} test_rule = {'es_host': 'test_host', 'es_port': 12345, @@ -45,18 +42,20 @@ test_args.config = 'test_config' test_args.rule = None test_args.debug = False +test_args.es_debug_trace = None def test_import_rules(): + rules_loader = FileRulesLoader(test_config) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['type'] = 'testing.test.RuleType' - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'load_yaml') as mock_open: mock_open.return_value = test_rule_copy # Test that type is imported - with mock.patch('__builtin__.__import__') as mock_import: + with mock.patch('builtins.__import__') as mock_import: mock_import.return_value = elastalert.ruletypes - load_configuration('test_config', test_config) + rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing.test' assert mock_import.call_args_list[0][0][3] == ['RuleType'] @@ -64,14 +63,15 @@ def test_import_rules(): test_rule_copy = copy.deepcopy(test_rule) mock_open.return_value = test_rule_copy test_rule_copy['alert'] = 'testing2.test2.Alerter' - with mock.patch('__builtin__.__import__') as mock_import: + with mock.patch('builtins.__import__') as mock_import: mock_import.return_value = elastalert.alerts - load_configuration('test_config', test_config) + rules_loader.load_configuration('test_config', test_config) assert mock_import.call_args_list[0][0][0] == 'testing2.test2' assert mock_import.call_args_list[0][0][3] == ['Alerter'] def test_import_import(): + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -82,9 +82,9 @@ def test_import_import(): 'email': 'ignored@email', # overwritten by the email in import_rule } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert mock_open.call_args_list[0][0] == ('blah.yaml',) assert mock_open.call_args_list[1][0] == ('importme.ymlt',) assert len(mock_open.call_args_list) == 2 @@ -94,10 +94,11 @@ def test_import_import(): assert rules['filter'] == import_rule['filter'] # check global import_rule dependency - assert import_rules == {'blah.yaml': ['importme.ymlt']} + assert rules_loader.import_rules == {'blah.yaml': ['importme.ymlt']} def test_import_absolute_import(): + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -108,9 +109,9 @@ def test_import_absolute_import(): 'email': 'ignored@email', # overwritten by the email in import_rule } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert mock_open.call_args_list[0][0] == ('blah.yaml',) assert mock_open.call_args_list[1][0] == ('/importme.ymlt',) assert len(mock_open.call_args_list) == 2 @@ -123,6 +124,7 @@ def test_import_absolute_import(): def test_import_filter(): # Check that if a filter is specified the rules are merged: + rules_loader = FileRulesLoader(test_config) import_rule = copy.deepcopy(test_rule) del(import_rule['es_host']) del(import_rule['es_port']) @@ -133,13 +135,14 @@ def test_import_filter(): 'filter': [{'term': {'ratchet': 'clank'}}], } - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [import_rule, import_me] - rules = load_configuration('blah.yaml', test_config) + rules = rules_loader.load_configuration('blah.yaml', test_config) assert rules['filter'] == [{'term': {'ratchet': 'clank'}}, {'term': {'key': 'value'}}] def test_load_inline_alert_rule(): + rules_loader = FileRulesLoader(test_config) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['alert'] = [ { @@ -154,34 +157,75 @@ def test_load_inline_alert_rule(): } ] test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.side_effect = [test_config_copy, test_rule_copy] - load_modules(test_rule_copy) + rules_loader.load_modules(test_rule_copy) assert isinstance(test_rule_copy['alert'][0], elastalert.alerts.EmailAlerter) assert isinstance(test_rule_copy['alert'][1], elastalert.alerts.EmailAlerter) assert 'foo@bar.baz' in test_rule_copy['alert'][0].rule['email'] assert 'baz@foo.bar' in test_rule_copy['alert'][1].rule['email'] +def test_file_rules_loader_get_names_recursive(): + conf = {'scan_subdirectories': True, 'rules_folder': 'root'} + rules_loader = FileRulesLoader(conf) + walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)), + ('root/folder_a', (), ('a.yaml', 'ab.yaml')), + ('root/folder_b', (), ('b.yaml',))) + with mock.patch('os.walk') as mock_walk: + mock_walk.return_value = walk_paths + paths = rules_loader.get_names(conf) + + paths = [p.replace(os.path.sep, '/') for p in paths] + + assert 'root/rule.yaml' in paths + assert 'root/folder_a/a.yaml' in paths + assert 'root/folder_a/ab.yaml' in paths + assert 'root/folder_b/b.yaml' in paths + assert len(paths) == 4 + + +def test_file_rules_loader_get_names(): + # Check for no subdirectory + conf = {'scan_subdirectories': False, 'rules_folder': 'root'} + rules_loader = FileRulesLoader(conf) + files = ['badfile', 'a.yaml', 'b.yaml'] + + with mock.patch('os.listdir') as mock_list: + with mock.patch('os.path.isfile') as mock_path: + mock_path.return_value = True + mock_list.return_value = files + paths = rules_loader.get_names(conf) + + paths = [p.replace(os.path.sep, '/') for p in paths] + + assert 'root/a.yaml' in paths + assert 'root/b.yaml' in paths + assert len(paths) == 2 + + def test_load_rules(): test_rule_copy = copy.deepcopy(test_rule) test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] - - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) - assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) - assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter) - assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta) - assert isinstance(rules['run_every'], datetime.timedelta) - for included_key in ['comparekey', 'testkey', '@timestamp']: - assert included_key in rules['rules'][0]['include'] - - # Assert include doesn't contain duplicates - assert rules['rules'][0]['include'].count('@timestamp') == 1 - assert rules['rules'][0]['include'].count('comparekey') == 1 + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + + with mock.patch('os.walk') as mock_ls: + mock_ls.return_value = [('', [], ['testrule.yaml'])] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + assert isinstance(rules['rules'][0]['type'], elastalert.ruletypes.RuleType) + assert isinstance(rules['rules'][0]['alert'][0], elastalert.alerts.Alerter) + assert isinstance(rules['rules'][0]['timeframe'], datetime.timedelta) + assert isinstance(rules['run_every'], datetime.timedelta) + for included_key in ['comparekey', 'testkey', '@timestamp']: + assert included_key in rules['rules'][0]['include'] + + # Assert include doesn't contain duplicates + assert rules['rules'][0]['include'].count('@timestamp') == 1 + assert rules['rules'][0]['include'].count('comparekey') == 1 def test_load_default_host_port(): @@ -189,16 +233,19 @@ def test_load_default_host_port(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.walk') as mock_ls: + mock_ls.return_value = [('', [], ['testrule.yaml'])] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - # Assert include doesn't contain duplicates - assert rules['es_port'] == 12345 - assert rules['es_host'] == 'elasticsearch.test' + # Assert include doesn't contain duplicates + assert rules['es_port'] == 12345 + assert rules['es_host'] == 'elasticsearch.test' def test_load_ssl_env_false(): @@ -206,15 +253,18 @@ def test_load_ssl_env_false(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_USE_SSL': 'false'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_USE_SSL': 'false'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['use_ssl'] is False + assert rules['use_ssl'] is False def test_load_ssl_env_true(): @@ -222,15 +272,18 @@ def test_load_ssl_env_true(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_USE_SSL': 'true'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_USE_SSL': 'true'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['use_ssl'] is True + assert rules['use_ssl'] is True def test_load_url_prefix_env(): @@ -238,68 +291,103 @@ def test_load_url_prefix_env(): test_rule_copy.pop('es_host') test_rule_copy.pop('es_port') test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy - with mock.patch('os.listdir') as mock_ls: - with mock.patch.dict(os.environ, {'ES_URL_PREFIX': 'es/'}): - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) + with mock.patch('os.listdir') as mock_ls: + with mock.patch.dict(os.environ, {'ES_URL_PREFIX': 'es/'}): + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) - assert rules['es_url_prefix'] == 'es/' + assert rules['es_url_prefix'] == 'es/' def test_load_disabled_rules(): test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['is_enabled'] = False test_config_copy = copy.deepcopy(test_config) - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + + with mock.patch('os.listdir') as mock_ls: + mock_ls.return_value = ['testrule.yaml'] + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) + # The rule is not loaded for it has "is_enabled=False" + assert len(rules['rules']) == 0 - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - rules = load_rules(test_args) - # The rule is not loaded for it has "is_enabled=False" - assert len(rules['rules']) == 0 + +def test_raises_on_missing_config(): + optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') + test_rule_copy = copy.deepcopy(test_rule) + for key in list(test_rule_copy.keys()): + test_rule_copy = copy.deepcopy(test_rule) + test_config_copy = copy.deepcopy(test_config) + test_rule_copy.pop(key) + + # Non required keys + if key in optional_keys: + continue + + with mock.patch('elastalert.config.yaml_loader') as mock_conf_open: + mock_conf_open.return_value = test_config_copy + with mock.patch('elastalert.loaders.yaml_loader') as mock_rule_open: + mock_rule_open.return_value = test_rule_copy + with mock.patch('os.walk') as mock_walk: + mock_walk.return_value = [('', [], ['testrule.yaml'])] + with pytest.raises(EAException, message='key %s should be required' % key): + rules = load_conf(test_args) + rules['rules'] = rules['rules_loader'].load(rules) def test_compound_query_key(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) test_rule_copy.pop('use_count_query') test_rule_copy['query_key'] = ['field1', 'field2'] - load_options(test_rule_copy, test_config, 'filename.yaml') + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') assert 'field1' in test_rule_copy['include'] assert 'field2' in test_rule_copy['include'] assert test_rule_copy['query_key'] == 'field1,field2' assert test_rule_copy['compound_query_key'] == ['field1', 'field2'] -def test_name_inference(): +def test_query_key_with_single_value(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) - test_rule_copy.pop('name') - load_options(test_rule_copy, test_config, 'msmerc woz ere.yaml') - assert test_rule_copy['name'] == 'msmerc woz ere' + test_rule_copy.pop('use_count_query') + test_rule_copy['query_key'] = ['field1'] + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert 'field1' in test_rule_copy['include'] + assert test_rule_copy['query_key'] == 'field1' + assert 'compound_query_key' not in test_rule_copy -def test_raises_on_missing_config(): - optional_keys = ('aggregation', 'use_count_query', 'query_key', 'compare_key', 'filter', 'include', 'es_host', 'es_port', 'name') +def test_query_key_with_no_values(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) test_rule_copy = copy.deepcopy(test_rule) - for key in test_rule_copy.keys(): - test_rule_copy = copy.deepcopy(test_rule) - test_config_copy = copy.deepcopy(test_config) - test_rule_copy.pop(key) + test_rule_copy.pop('use_count_query') + test_rule_copy['query_key'] = [] + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert 'query_key' not in test_rule_copy + assert 'compound_query_key' not in test_rule_copy - # Non required keys - if key in optional_keys: - continue - with mock.patch('elastalert.config.yaml_loader') as mock_open: - mock_open.side_effect = [test_config_copy, test_rule_copy] - with mock.patch('os.listdir') as mock_ls: - mock_ls.return_value = ['testrule.yaml'] - with pytest.raises(EAException, message='key %s should be required' % key): - rule = load_rules(test_args) - print(rule) +def test_name_inference(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy.pop('name') + rules_loader.load_options(test_rule_copy, test_config, 'msmerc woz ere.yaml') + assert test_rule_copy['name'] == 'msmerc woz ere' def test_raises_on_bad_generate_kibana_filters(): @@ -318,48 +406,35 @@ def test_raises_on_bad_generate_kibana_filters(): # Test that all the good filters work, but fail with a bad filter added for good in good_filters: + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) test_rule_copy['filter'] = good - with mock.patch('elastalert.config.yaml_loader') as mock_open: + with mock.patch.object(rules_loader, 'get_yaml') as mock_open: mock_open.return_value = test_rule_copy - load_configuration('blah', test_config) + rules_loader.load_configuration('blah', test_config) for bad in bad_filters: test_rule_copy['filter'] = good + bad with pytest.raises(EAException): - load_configuration('blah', test_config) - - -def test_get_file_paths_recursive(): - conf = {'scan_subdirectories': True, 'rules_folder': 'root'} - walk_paths = (('root', ('folder_a', 'folder_b'), ('rule.yaml',)), - ('root/folder_a', (), ('a.yaml', 'ab.yaml')), - ('root/folder_b', (), ('b.yaml',))) - with mock.patch('os.walk') as mock_walk: - mock_walk.return_value = walk_paths - paths = get_file_paths(conf) - - paths = [p.replace(os.path.sep, '/') for p in paths] - - assert 'root/rule.yaml' in paths - assert 'root/folder_a/a.yaml' in paths - assert 'root/folder_a/ab.yaml' in paths - assert 'root/folder_b/b.yaml' in paths - assert len(paths) == 4 - + rules_loader.load_configuration('blah', test_config) -def test_get_file_paths(): - # Check for no subdirectory - conf = {'scan_subdirectories': False, 'rules_folder': 'root'} - files = ['badfile', 'a.yaml', 'b.yaml'] - with mock.patch('os.listdir') as mock_list: - with mock.patch('os.path.isfile') as mock_path: - mock_path.return_value = True - mock_list.return_value = files - paths = get_file_paths(conf) +def test_kibana_discover_from_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['kibana_discover_from_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['kibana_discover_from_timedelta'], datetime.timedelta) + assert test_rule_copy['kibana_discover_from_timedelta'] == datetime.timedelta(minutes=2) - paths = [p.replace(os.path.sep, '/') for p in paths] - assert 'root/a.yaml' in paths - assert 'root/b.yaml' in paths - assert len(paths) == 2 +def test_kibana_discover_to_timedelta(): + test_config_copy = copy.deepcopy(test_config) + rules_loader = FileRulesLoader(test_config_copy) + test_rule_copy = copy.deepcopy(test_rule) + test_rule_copy['kibana_discover_to_timedelta'] = {'minutes': 2} + rules_loader.load_options(test_rule_copy, test_config, 'filename.yaml') + assert isinstance(test_rule_copy['kibana_discover_to_timedelta'], datetime.timedelta) + assert test_rule_copy['kibana_discover_to_timedelta'] == datetime.timedelta(minutes=2) diff --git a/tests/rules_test.py b/tests/rules_test.py index e08646a38..1954b5d54 100644 --- a/tests/rules_test.py +++ b/tests/rules_test.py @@ -1138,7 +1138,7 @@ def test_metric_aggregation(): rule = MetricAggregationRule(rules) - assert rule.rules['aggregation_query_element'] == {'cpu_pct_avg': {'avg': {'field': 'cpu_pct'}}} + assert rule.rules['aggregation_query_element'] == {'metric_cpu_pct_avg': {'avg': {'field': 'cpu_pct'}}} assert rule.crossed_thresholds(None) is False assert rule.crossed_thresholds(0.09) is True @@ -1146,17 +1146,17 @@ def test_metric_aggregation(): assert rule.crossed_thresholds(0.79) is False assert rule.crossed_thresholds(0.81) is True - rule.check_matches(datetime.datetime.now(), None, {'cpu_pct_avg': {'value': None}}) - rule.check_matches(datetime.datetime.now(), None, {'cpu_pct_avg': {'value': 0.5}}) + rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': None}}) + rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.5}}) assert len(rule.matches) == 0 - rule.check_matches(datetime.datetime.now(), None, {'cpu_pct_avg': {'value': 0.05}}) - rule.check_matches(datetime.datetime.now(), None, {'cpu_pct_avg': {'value': 0.95}}) + rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.05}}) + rule.check_matches(datetime.datetime.now(), None, {'metric_cpu_pct_avg': {'value': 0.95}}) assert len(rule.matches) == 2 rules['query_key'] = 'qk' rule = MetricAggregationRule(rules) - rule.check_matches(datetime.datetime.now(), 'qk_val', {'cpu_pct_avg': {'value': 0.95}}) + rule.check_matches(datetime.datetime.now(), 'qk_val', {'metric_cpu_pct_avg': {'value': 0.95}}) assert rule.matches[0]['qk'] == 'qk_val' @@ -1170,9 +1170,9 @@ def test_metric_aggregation_complex_query_key(): 'max_threshold': 0.8} query = {"bucket_aggs": {"buckets": [ - {"cpu_pct_avg": {"value": 0.91}, "key": "sub_qk_val1"}, - {"cpu_pct_avg": {"value": 0.95}, "key": "sub_qk_val2"}, - {"cpu_pct_avg": {"value": 0.89}, "key": "sub_qk_val3"}] + {"metric_cpu_pct_avg": {"value": 0.91}, "key": "sub_qk_val1"}, + {"metric_cpu_pct_avg": {"value": 0.95}, "key": "sub_qk_val2"}, + {"metric_cpu_pct_avg": {"value": 0.89}, "key": "sub_qk_val3"}] }, "key": "qk_val"} rule = MetricAggregationRule(rules) diff --git a/tests/util_test.py b/tests/util_test.py index 0f8d1d6a1..55a2f9c8f 100644 --- a/tests/util_test.py +++ b/tests/util_test.py @@ -14,6 +14,7 @@ from elastalert.util import replace_dots_in_field_names from elastalert.util import resolve_string from elastalert.util import set_es_key +from elastalert.util import should_scrolling_continue @pytest.mark.parametrize('spec, expected_delta', [ @@ -105,6 +106,24 @@ def test_looking_up_nested_composite_keys(ea): assert lookup_es_key(record, 'Fields.ts.value') == expected +def test_looking_up_arrays(ea): + record = { + 'flags': [1, 2, 3], + 'objects': [ + {'foo': 'bar'}, + {'foo': [{'bar': 'baz'}]}, + {'foo': {'bar': 'baz'}} + ] + } + assert lookup_es_key(record, 'flags[0]') == 1 + assert lookup_es_key(record, 'flags[1]') == 2 + assert lookup_es_key(record, 'objects[0]foo') == 'bar' + assert lookup_es_key(record, 'objects[1]foo[0]bar') == 'baz' + assert lookup_es_key(record, 'objects[2]foo.bar') == 'baz' + assert lookup_es_key(record, 'objects[1]foo[1]bar') is None + assert lookup_es_key(record, 'objects[1]foo[0]baz') is None + + def test_add_raw_postfix(ea): expected = 'foo.raw' assert add_raw_postfix('foo', False) == expected @@ -195,3 +214,17 @@ def test_format_index(): 'logstash-2018.06.25', 'logstash-2018.06.26'] assert sorted(format_index(pattern2, date, date2, True).split(',')) == ['logstash-2018.25', 'logstash-2018.26'] + + +def test_should_scrolling_continue(): + rule_no_max_scrolling = {'max_scrolling_count': 0, 'scrolling_cycle': 1} + rule_reached_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 2} + rule_before_first_run = {'max_scrolling_count': 0, 'scrolling_cycle': 0} + rule_before_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 1} + rule_over_max_scrolling = {'max_scrolling_count': 2, 'scrolling_cycle': 3} + + assert should_scrolling_continue(rule_no_max_scrolling) is True + assert should_scrolling_continue(rule_reached_max_scrolling) is False + assert should_scrolling_continue(rule_before_first_run) is True + assert should_scrolling_continue(rule_before_max_scrolling) is True + assert should_scrolling_continue(rule_over_max_scrolling) is False diff --git a/tox.ini b/tox.ini index e3bb2b945..71099e17c 100644 --- a/tox.ini +++ b/tox.ini @@ -1,6 +1,6 @@ [tox] project = elastalert -envlist = py27,docs +envlist = py36,docs [testenv] deps = -rrequirements-dev.txt @@ -25,6 +25,6 @@ norecursedirs = .* virtualenv_run docs build venv env [testenv:docs] deps = {[testenv]deps} - sphinx + sphinx==1.6.6 changedir = docs -commands = sphinx-build -b html -d build/doctrees source build/html +commands = sphinx-build -b html -d build/doctrees -W source build/html