Merge branch 'release/0.0.9'

Author: Ruben van Vreeland

.gitignore

@@ -71,4 +71,5 @@ jspm_packages
 .node_repl_history
 
 lib/
+*.pyc
 config/config.json

.gitlab-ci.yml

@@ -1,3 +1,24 @@
+# Cache template
+.default-cache: &default-cache
+  key: elastalert
+  paths:
+    - node_modules
+
+.push-cache: &push-cache
+  cache:
+    <<: *default-cache
+    policy: push
+
+.pull-cache: &pull-cache
+  cache:
+    <<: *default-cache
+    policy: pull
+
+.node_test: &node-test
+  <<: *pull-cache
+  image: node:slim
+  stage: test
+
 stages:
   - build
   - test
@@ -11,36 +32,30 @@ cache:
   policy: pull
 
 build:
+  <<: *push-cache
   image: node:slim
   stage: build
-  cache:
-    key: elastalert
-    paths:
-      - node_modules/
-    policy: push
   script:
-    - npm i -q
+    - npm install --quiet
     - npm run build
 
 test:
-  image: node:slim
-  stage: test
+  <<: *node-test
   script:
-    - npm i -q
+    - npm install --quiet
     - npm test
 
 lint:
-  image: node:slim
-  allow_failure: true
-  stage: test
+  <<: *node-test
   script:
-    - npm i -q
+    - npm install --quiet
     - ./node_modules/.bin/eslint .
 
 deploy:npm:
   image: node:slim
   stage: deploy
   script:
+    - npm install --quiet
     - scripts/update-authors.sh
     - npm publish --access public
   only:
@@ -82,3 +97,6 @@ mirror:github:
     - git remote add github https://$MIRROR_GITHUB_USER:$MIRROR_GITHUB_PASSWORD@$MIRROR_GITHUB_URL
     - git push -u github -q --mirror
   when: always
+  only:
+    - tags
+    - develop

Dockerfile

@@ -1,23 +1,24 @@
 FROM ivankrizsan/elastalert AS py-ea
-# Uses BitSensor's elastalert 
-ENV ELASTALERT_URL https://github.com/bitsensor/yelp-elastalert/archive/master.zip 
+# Uses BitSensor's elastalert, or yelp master if commented out
+# ENV ELASTALERT_URL https://github.com/bitsensor/yelp-elastalert/archive/master.zip
 
 FROM node:alpine
 MAINTAINER BitSensor <[email protected]>
 EXPOSE 3030
- 
+
 RUN apk update && apk upgrade && apk add python2
 
 COPY --from=py-ea /usr/lib/python2.7/site-packages /usr/lib/python2.7/site-packages
 COPY --from=py-ea /opt/elastalert /opt/elastalert
 
 RUN mkdir server_data
-WORKDIR /opt/elastalert-server 
-COPY . /opt/elastalert-server 
- 
-RUN npm install --production --quiet 
-COPY config/elastalert.yaml /opt/elastalert/config.yaml 
-COPY config/config.json config/config.json 
+WORKDIR /opt/elastalert-server
+COPY . /opt/elastalert-server
+
+RUN npm install --production --quiet
+COPY config/elastalert.yaml /opt/elastalert/config.yaml
+COPY config/config.json config/config.json
 COPY rule_templates/ /opt/elastalert/rule_templates
+COPY elastalert_modules/ /opt/elastalert/elastalert_modules
 
-ENTRYPOINT ["npm", "start"]
+ENTRYPOINT ["npm", "start"]

README.md

@@ -18,6 +18,7 @@ docker run -d -p 3030:3030 \
     -v `pwd`/config/config.json:/opt/elastalert-server/config/config.json \
     -v `pwd`/rules:/opt/elastalert/rules \
     -v `pwd`/rule_templates:/opt/elastalert/rule_templates \
+    -v `pwd`/elastalert_modules:/opt/elastalert/elastalert_modules \
     --net="host" \
     --name elastalert elastalert:latest
 ```
@@ -29,13 +30,16 @@ docker run -d -p 3030:3030 \
     -v (pwd)/config/config.json:/opt/elastalert-server/config/config.json \
     -v (pwd)/rules:/opt/elastalert/rules \
     -v (pwd)/rule_templates:/opt/elastalert/rule_templates \
+    -v (pwd)/elastalert_modules:/opt/elastalert/elastalert_modules \
     --net="host" \
     --name elastalert elastalert:latest
 ```
 ### Configuration
 #### ElastAlert parameters
 ElastAlert supports additional arguments, that can be passed in the `config.json` file. An example is given in `config/config-historic-data-example.json`. 
 
+
+
 ## Installation using npm and manual ElastAlert setup
 
 ### Requirements

config/elastalert.yaml

@@ -12,12 +12,12 @@ rules_folder: rules
 # How often ElastAlert will query elasticsearch
 # The unit can be anything from weeks to seconds
 run_every:
-  seconds: 1
+  seconds: 5
 
 # ElastAlert will buffer results from the most recent
 # period of time, in case some log sources are not in real time
 buffer_time:
-  minutes: 15
+  minutes: 1
 
 # Optional URL prefix for elasticsearch
 #es_url_prefix: elasticsearch

elastalert_modules/__init__.py

@@ -0,0 +1,13 @@
+from elastalert.enhancements import BaseEnhancement
+from util import convert_array_to_object
+from util import parse_detections
+
+# For easier access to nested values in an array , this merges all items in  array
+# within  
+class AlertTextEnhancement(BaseEnhancement):
+    # The enhancement is run against every match
+    # The match is passed to the process function where it can be modified in any way
+    # ElastAlert will do this for each enhancement linked to a rule
+    def process(self, match):
+        parsed_match = parse_detections(match)
+        match.update(parsed_match)

elastalert_modules/bitsensor_enhancement.py

@@ -0,0 +1,30 @@
+from itertools import chain
+
+def convert_array_to_object(array):
+    json = {}
+    for idx in range(len(array)):
+        json[str(idx)] = array[idx]
+    return json
+
+def parse_detections(match):
+    key = 'detections'
+    parsed = {key+'_parsed': {}}
+
+    if not isinstance(match[key], list):
+        return parsed
+    if len(match[key]) == 0:
+        return parsed
+ 
+    # Converts array terms into objects 
+    # parsed[key + '_parsed'] = convert_array_to_object(match[key]) 
+
+    for sk, value in match[key][0].iteritems(): 
+        value_array = [] 
+        if isinstance(value, list):
+            value_array = list(chain.from_iterable(sv for sv in (v[sk] for v in match[key]) if sv))
+        else:
+            value_array = [v[sk] for v in match[key]] 
+        unique_values = set(value_array)
+        parsed[key + '_parsed'][sk] = ", ".join(str(va) for va in unique_values)
+ 
+    return parsed

elastalert_modules/util.py

@@ -1,6 +1,6 @@
 {
   "name": "@bitsensor/elastalert",
-  "version": "0.0.8",
+  "version": "0.0.9",
   "description": "A server that runs ElastAlert and exposes REST API's for manipulating rules and alerts.",
   "license": "MIT",
   "main": "index.js",

package.json

@@ -24,20 +24,28 @@ include:
   - context.http.userAgent
   - context.ip
   - context.php.session.sessionId
-  - detections.type
-  - detections.name
+  - detections
   - meta.user
 
-alert_subject: "Detection on {}"
+
+# Enhancement for converting 'detections' array into object, ex. get merged detection type by  
+# 'detections_parsed.type' or get first detection type by 'detection_parsed.0.type' 
+match_enhancements:   
+- "elastalert_modules.bitsensor_enhancement.AlertTextEnhancement"   
+run_enhancements_first: true 
+
+
+alert_subject: ":exclamation: Detection on {}"
 alert_subject_args:
   - endpoint.name
 
 alert_text_type: alert_text_only
-alert_text: "Detection triggered at {}\n\nAttacker:\nIP: {} \nUser-Agent: {}\n\n:Id: {}\nUser: {}"
+alert_text: "Triggered at _{}_\n\n*Attacker:*\nIP: {} \nUser-Agent: {}\nDetection: `{}`\n\n:Id: {}\nUser: {}"
 alert_text_args:
   - endpoint.localtime
   - context.ip
   - context.http.userAgent
+  - detections_parsed.type
   - _id
   - meta.user