bitsensor · backstroke-bot · Nov 1, 2018 · Nov 13, 2018 · Nov 13, 2018 · Nov 14, 2018
diff --git a/.editorconfig b/.editorconfig
@@ -0,0 +1,17 @@
+root = true
+
+[*]
+end_of_line = lf
+insert_final_newline = true
+charset = utf-8
+
+[*.py]
+indent_style = space
+indent_size = 4
+
+[Makefile]
+indent_style = tab
+
+[{*.json,*.yml,*.yaml}]
+indent_style = space
+indent_size = 2
diff --git a/.travis.yml b/.travis.yml
@@ -1,12 +1,35 @@
 language: python
 python:
-- '2.7'
+- '3.6'
 env:
 - TOXENV=docs
-- TOXENV=py27
+- TOXENV=py36
 install:
 - pip install tox
-script: make test
+- >
+  if [[ -n "${ES_VERSION}" ]] ; then
+    wget https://artifacts.elastic.co/downloads/elasticsearch/elasticsearch-${ES_VERSION}.tar.gz
+    mkdir elasticsearch-${ES_VERSION} && tar -xzf elasticsearch-${ES_VERSION}.tar.gz -C elasticsearch-${ES_VERSION} --strip-components=1
+    ./elasticsearch-${ES_VERSION}/bin/elasticsearch &
+  fi
+script:
+- >
+  if [[ -n "${ES_VERSION}" ]] ; then
+    wget -q --waitretry=1 --retry-connrefused --tries=30 -O - http://127.0.0.1:9200
+    make test-elasticsearch
+  else
+    make test
+  fi
+jobs:
+  include:
+    - stage: 'Elasticsearch test'
+      env: TOXENV=py36 ES_VERSION=7.0.0-linux-x86_64
+    - env: TOXENV=py36 ES_VERSION=6.6.2
+    - env: TOXENV=py36 ES_VERSION=6.3.2
+    - env: TOXENV=py36 ES_VERSION=6.2.4
+    - env: TOXENV=py36 ES_VERSION=6.0.1
+    - env: TOXENV=py36 ES_VERSION=5.6.16
+
 deploy:
   provider: pypi
   user: yelplabs

diff --git a/Dockerfile-test b/Dockerfile-test
@@ -1,11 +1,9 @@
 FROM ubuntu:latest
 
 RUN apt-get update && apt-get upgrade -y
-RUN apt-get -y install build-essential python-setuptools python2.7 python2.7-dev libssl-dev git tox
-
-RUN easy_install pip
+RUN apt-get -y install build-essential python3.6 python3.6-dev python3-pip libssl-dev git
 
 WORKDIR /home/elastalert
 
 ADD requirements*.txt ./
-RUN pip install -r requirements-dev.txt
+RUN pip3 install -r requirements-dev.txt
diff --git a/Makefile b/Makefile
@@ -16,6 +16,9 @@ install-hooks:
 test:
 	tox
 
+test-elasticsearch:
+	tox -- --runelasticsearch
+
 test-docker:
 	docker-compose --project-name elastalert build tox
 	docker-compose --project-name elastalert run tox

diff --git a/README.md b/README.md
@@ -1,5 +1,6 @@
-[![Stories in Ready](https://badge.waffle.io/Yelp/elastalert.png?label=ready&title=Ready)](https://waffle.io/Yelp/elastalert)
-[![Stories in In Progress](https://badge.waffle.io/Yelp/elastalert.png?label=in%20progress&title=In%20Progress)](https://waffle.io/Yelp/elastalert)
+**ElastAlert is no longer maintained. Please use [ElastAlert2](https://github.com/jertel/elastalert2) instead.**
+
+
 [![Build Status](https://travis-ci.org/Yelp/elastalert.svg)](https://travis-ci.org/Yelp/elastalert)
 [![Join the chat at https://gitter.im/Yelp/elastalert](https://badges.gitter.im/Join%20Chat.svg)](https://gitter.im/Yelp/elastalert?utm_source=badge&utm_medium=badge&utm_campaign=pr-badge&utm_content=badge)
 
@@ -47,12 +48,16 @@ Currently, we have built-in support for the following alert types:
 - MS Teams
 - Slack
 - Telegram
+- GoogleChat
 - AWS SNS
 - VictorOps
 - PagerDuty
+- PagerTree
 - Exotel
 - Twilio
 - Gitter
+- Line Notify
+- Zabbix
 
 Additional rule types and alerts can be easily imported or written.
 
@@ -67,8 +72,23 @@ In addition to this basic usage, there are many other features that make alerts
 To get started, check out `Running ElastAlert For The First Time` in the [documentation](http://elastalert.readthedocs.org).
 
 ## Running ElastAlert
+You can either install the latest released version of ElastAlert using pip:
+
+```pip install elastalert```
+
+or you can clone the ElastAlert repository for the most recent changes:
+
+```git clone https://github.com/Yelp/elastalert.git```
+
+Install the module:
+
+```pip install "setuptools>=11.3"```
+
+```python setup.py install```
+
+The following invocation can be used to run ElastAlert after installing
 
-``$ python elastalert/elastalert.py [--debug] [--verbose] [--start <timestamp>] [--end <timestamp>] [--rule <filename.yaml>] [--config <filename.yaml>]``
+``$ elastalert [--debug] [--verbose] [--start <timestamp>] [--end <timestamp>] [--rule <filename.yaml>] [--config <filename.yaml>]``
 
 ``--debug`` will print additional information to the screen as well as suppresses alerts and instead prints the alert body. Not compatible with `--verbose`.
 
@@ -88,11 +108,11 @@ Eg: ``--rule this_rule.yaml``
 
 ## Third Party Tools And Extras
 ### Kibana plugin
-![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/kibana-elastalert-plugin-showcase.gif)
+![img](https://raw.githubusercontent.com/bitsensor/elastalert-kibana-plugin/master/showcase.gif)
 Available at the [ElastAlert Kibana plugin repository](https://github.com/bitsensor/elastalert-kibana-plugin).
 
 ### Docker
-A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`. 
+A [Dockerized version](https://github.com/bitsensor/elastalert) of ElastAlert including a REST api is build from `master` to `bitsensor/elastalert:latest`.
 
 ```bash
 git clone https://github.com/bitsensor/elastalert.git; cd elastalert

diff --git a/changelog.md b/changelog.md
@@ -1,5 +1,120 @@
 # Change Log
 
+# v0.2.4
+
+### Added
+- Added back customFields support for The Hive
+
+# v0.2.3
+
+### Added
+- Added back TheHive alerter without TheHive4py library
+
+# v0.2.2
+
+### Added
+- Integration with Kibana Discover app
+- Addied ability to specify opsgenie alert details 
+
+### Fixed
+- Fix some encoding issues with command alerter
+- Better error messages for missing config file
+- Fixed an issue with run_every not applying per-rule
+- Fixed an issue with rules not being removed
+- Fixed an issue with top count keys and nested query keys
+- Various documentation fixes
+- Fixed an issue with not being able to use spike aggregation
+
+### Removed
+- Remove The Hive alerter
+
+# v0.2.1
+
+### Fixed
+- Fixed an AttributeError introduced in 0.2.0
+
+# v0.2.0
+
+- Switched to Python 3
+
+### Added
+- Add rule loader class for customized rule loading
+- Added thread based rules and limit_execution
+- Run_every can now be customized per rule
+
+### Fixed
+- Various small fixes
+
+# v0.1.39
+
+### Added
+- Added spike alerts for metric aggregations
+- Allow SSL connections for Stomp
+- Allow limits on alert text length
+- Add optional min doc count for terms queries
+- Add ability to index into arrays for alert_text_args, etc
+
+### Fixed
+- Fixed bug involving --config flag with create-index
+- Fixed some settings not being inherited from the config properly
+- Some fixes for Hive alerter
+- Close SMTP connections properly
+- Fix timestamps in Pagerduty v2 payload
+- Fixed an bug causing aggregated alerts to mix up
+
+# v0.1.38
+
+### Added
+- Added PagerTree alerter
+- Added Line alerter
+- Added more customizable logging
+- Added new logic in test-rule to detemine the default timeframe
+
+### Fixed
+- Fixed an issue causing buffer_time to sometimes be ignored
+
+# v0.1.37
+
+### Added
+- Added more options for Opsgenie alerter
+- Added more pagerduty options
+- Added ability to add metadata to elastalert logs
+
+### Fixed
+- Fixed some documentation to be more clear
+- Stop requiring doc_type for metric aggregations
+- No longer puts quotes around regex terms in blacklists or whitelists
+
+# v0.1.36
+
+### Added
+- Added a prefix "metric_" to the key used for metric aggregations to avoid possible conflicts
+- Added option to skip Alerta certificate validation
+
+### Fixed
+- Fixed a typo in the documentation for spike rule
+
+# v0.1.35
+
+### Fixed
+- Fixed an issue preventing new term rule from working with terms query
+
+# v0.1.34
+
+### Added
+- Added prefix/suffix support for summary table
+- Added support for ignoring SSL validation in Slack
+- More visible exceptions during query parse failures
+
+### Fixed
+- Fixed top_count_keys when using compound query_key
+- Fixed num_hits sometimes being reported too low
+- Fixed an issue with setting ES_USERNAME via env
+- Fixed an issue when using test script with custom timestamps
+- Fixed a unicode error when using Telegram
+- Fixed an issue with jsonschema version conflict
+- Fixed an issue with nested timestamps in cardinality type
+
 # v0.1.33
 
 ### Added

diff --git a/config.yaml.example b/config.yaml.example
@@ -57,8 +57,59 @@ es_port: 9200
 # This can be a unmapped index, but it is recommended that you run
 # elastalert-create-index to set a mapping
 writeback_index: elastalert_status
+writeback_alias: elastalert_alerts
 
 # If an alert fails for some reason, ElastAlert will retry
 # sending the alert until this time period has elapsed
 alert_time_limit:
   days: 2
+
+# Custom logging configuration
+# If you want to setup your own logging configuration to log into
+# files as well or to Logstash and/or modify log levels, use
+# the configuration below and adjust to your needs.
+# Note: if you run ElastAlert with --verbose/--debug, the log level of
+# the "elastalert" logger is changed to INFO, if not already INFO/DEBUG.
+#logging:
+#  version: 1
+#  incremental: false
+#  disable_existing_loggers: false
+#  formatters:
+#    logline:
+#      format: '%(asctime)s %(levelname)+8s %(name)+20s %(message)s'
+#
+#    handlers:
+#      console:
+#        class: logging.StreamHandler
+#        formatter: logline
+#        level: DEBUG
+#        stream: ext://sys.stderr
+#
+#      file:
+#        class : logging.FileHandler
+#        formatter: logline
+#        level: DEBUG
+#        filename: elastalert.log
+#
+#    loggers:
+#      elastalert:
+#        level: WARN
+#        handlers: []
+#        propagate: true
+#
+#      elasticsearch:
+#        level: WARN
+#        handlers: []
+#        propagate: true
+#
+#      elasticsearch.trace:
+#        level: WARN
+#        handlers: []
+#        propagate: true
+#
+#      '':  # root logger
+#        level: WARN
+#          handlers:
+#            - console
+#            - file
+#        propagate: false
diff --git a/docs/source/elastalert.rst b/docs/source/elastalert.rst
@@ -39,8 +39,10 @@ Currently, we have support built in for these alert types:
 - HipChat
 - Slack
 - Telegram
+- GoogleChat
 - Debug
 - Stomp
+- TheHive
 
 Additional rule types and alerts can be easily imported or written. (See :ref:`Writing rule types <writingrules>` and :ref:`Writing alerts <writingalerts>`)
 
@@ -131,9 +133,12 @@ The environment variable ``ES_USE_SSL`` will override this field.
 
 ``es_conn_timeout``: Optional; sets timeout for connecting to and reading from ``es_host``; defaults to ``20``.
 
+``rules_loader``: Optional; sets the loader class to be used by ElastAlert to retrieve rules and hashes.
+Defaults to ``FileRulesLoader`` if not set.
+
 ``rules_folder``: The name of the folder which contains rule configuration files. ElastAlert will load all
 files in this folder, and all subdirectories, that end in .yaml. If the contents of this folder change, ElastAlert will load, reload
-or remove rules based on their respective config files.
+or remove rules based on their respective config files. (only required when using ``FileRulesLoader``).
 
 ``scan_subdirectories``: Optional; Sets whether or not ElastAlert should recursively descend the rules directory - ``true`` or ``false``. The default is ``true``
 
@@ -146,7 +151,11 @@ configuration.
 
 ``max_query_size``: The maximum number of documents that will be downloaded from Elasticsearch in a single query. The
 default is 10,000, and if you expect to get near this number, consider using ``use_count_query`` for the rule. If this
-limit is reached, ElastAlert will `scroll <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html>`_ through pages the size of ``max_query_size`` until processing all results.
+limit is reached, ElastAlert will `scroll <https://www.elastic.co/guide/en/elasticsearch/reference/current/search-request-scroll.html>`_
+using the size of ``max_query_size`` through the set amount of pages, when ``max_scrolling_count`` is set or until processing all results.
+
+``max_scrolling_count``: The maximum amount of pages to scroll through. The default is ``0``, which means the scrolling has no limit.
+For example if this value is set to ``5`` and the ``max_query_size`` is set to ``10000`` then ``50000`` documents will be downloaded at most.
 
 ``scroll_keepalive``: The maximum time (formatted in `Time Units <https://www.elastic.co/guide/en/elasticsearch/reference/current/common-options.html#time-units>`_) the scrolling context should be kept alive. Avoid using high values as it abuses resources in Elasticsearch, but be mindful to allow sufficient time to finish processing all the results.
 
@@ -161,6 +170,8 @@ from that time, unless it is older than ``old_query_limit``, in which case it wi
 will upload a traceback message to ``elastalert_metadata`` and if ``notify_email`` is set, send an email notification. The
 rule will no longer be run until either ElastAlert restarts or the rule file has been modified. This defaults to True.
 
+``show_disabled_rules``: If true, ElastAlert show the disable rules' list when finishes the execution. This defaults to True.
+
 ``notify_email``: An email address, or list of email addresses, to which notification emails will be sent. Currently,
 only an uncaught exception will send a notification email. The from address, SMTP host, and reply-to header can be set
 using ``from_addr``, ``smtp_host``, and ``email_reply_to`` options, respectively. By default, no emails will be sent.
@@ -188,6 +199,24 @@ The default value is ``False``. Elasticsearch 2.0 - 2.3 does not support dots in
 ``string_multi_field_name``: If set, the suffix to use for the subfield for string multi-fields in Elasticsearch.
 The default value is ``.raw`` for Elasticsearch 2 and ``.keyword`` for Elasticsearch 5.
 
+``add_metadata_alert``: If set, alerts will include metadata described in rules (``category``, ``description``, ``owner`` and ``priority``); set to ``True`` or ``False``. The default is ``False``.
+
+``skip_invalid``: If ``True``, skip invalid files instead of exiting.
+
+Logging
+-------
+
+By default, ElastAlert uses a simple basic logging configuration to print log messages to standard error.
+You can change the log level to ``INFO`` messages by using the ``--verbose`` or ``--debug`` command line options.
+
+If you need a more sophisticated logging configuration, you can provide a full logging configuration
+in the config file. This way you can also configure logging to a file, to Logstash and
+adjust the logging format.
+
+For details, see the end of ``config.yaml.example`` where you can find an example logging
+configuration.
+
+
 .. _runningelastalert:
 
 Running ElastAlert