Merge branch 'ServerCentral-json-results' into develop

Author: martijnrondeel

README.md

@@ -111,7 +111,10 @@ You can use the following config options:
   "dataPath": { // The path to a folder that the server can use to store data and temporary files.
     "relative": true, // Whether to use a path relative to the `elastalertPath` folder.
     "path": "/server_data" // The path to the data folder.
-  }
+  },
+  "es_host": "elastalert", // For getting metadata and field mappings, connect to this ES server
+  "es_port": 9200, // Port for above
+  "writeback_index": "elastalert_status" // Writeback index to examine for /metadata endpoint
 }
 ```
 
@@ -200,11 +203,25 @@ This server exposes the following REST API's:
           "days": "1"
           
           // Whether to send real alerts
-          "alert": false
+          "alert": false,
+
+          // Return results in structured JSON
+          "format": "json",
+
+          // Limit returned results to this amount
+          "maxResults": 1000
         }
       }
       ``` 
 
+- **GET `/metadata/:type`**
+
+    Returns metadata from elasticsearch related to elasalert's state. `:type` should be one of: elastalert_status, elastalert, elastalert_error, or silence. See [docs about the elastalert metadata index](https://elastalert.readthedocs.io/en/latest/elastalert_status.html).
+
+- **GET `/mapping/:index`**
+
+    Returns field mapping from elasticsearch for a given index. 
+
 - **[WIP] GET `/config`**
 
     Gets the ElastAlert configuration from `config.yaml` in `elastalertPath` (from the config).

config/config.json

@@ -12,5 +12,8 @@
   "templatesPath": {
     "relative": true,
     "path": "/rule_templates"
-  }
-}
+  },
+  "es_host": "elastalert",
+  "es_port": 9200,
+  "writeback_index": "elastalert_status"
+}

package.json

@@ -23,6 +23,8 @@
     "babel-register": "^6.14.0",
     "body-parser": "^1.15.2",
     "bunyan": "^1.8.1",
+    "cors": "^2.8.4",
+    "elasticsearch": "^15.1.1",
     "express": "^4.14.0",
     "fs-extra": "^5.0.0",
     "joi": "^13.1.2",

src/common/config/schema.js

@@ -3,6 +3,9 @@ import Joi from 'joi';
 
 const schema = Joi.object().keys({
   'appName': Joi.string().default('elastalert-server'),
+  'es_host': Joi.string().default('elastalert'),
+  'es_port': Joi.number().default(9200),
+  'writeback_index': Joi.string().default('elastalert_status'),
   'port': Joi.number().default(3030),
   'elastalertPath': Joi.string().default('/opt/elastalert'),
   'rulesPath': Joi.object().keys({

src/common/elasticsearch_client.js

@@ -0,0 +1,9 @@
+import elasticsearch from 'elasticsearch';
+import config from './config';
+
+export function getClient() {
+  var client = new elasticsearch.Client({
+    hosts: [ `http://${config.get('es_host')}:${config.get('es_port')}`]
+  });
+  return client;
+}

src/controllers/test/index.js

@@ -33,6 +33,15 @@ export default class TestController {
 
           processOptions.push('-m', 'elastalert.test_rule', '--config', 'config.yaml', tempFilePath, '--days', options.days);
 
+          if (options.format === 'json') {
+            processOptions.push('--formatted-output');
+          }
+
+          if (options.maxResults > 0) {
+            processOptions.push('--max-query-size');
+            processOptions.push(options.maxResults);
+          }
+
           if (options.alert) {
             processOptions.push('--alert');
           }
@@ -61,7 +70,12 @@ export default class TestController {
 
             testProcess.on('exit', function (statusCode) {
               if (statusCode === 0) {
-                resolve(stdoutLines.join('\n'));
+                if (options.format === 'json') {
+                  resolve(stdoutLines.join(''));
+                }
+                else {
+                  resolve(stdoutLines.join('\n'));
+                }
               } else {
                 reject(stderrLines.join('\n'));
                 logger.error(stderrLines.join('\n'));

src/elastalert_server.js

@@ -9,6 +9,7 @@ import ProcessController from './controllers/process';
 import RulesController from './controllers/rules';
 import TemplatesController from './controllers/templates';
 import TestController from './controllers/test';
+import cors from 'cors';
 
 let logger = new Logger('Server');
 
@@ -58,6 +59,7 @@ export default class ElastalertServer {
     // Start the server when the config is loaded
     config.ready(function () {
       try {
+        self._express.use(cors());
         self._express.use(bodyParser.json());
         self._express.use(bodyParser.urlencoded({ extended: true }));
         self._setupRouter();

src/handlers/mapping/get.js

@@ -0,0 +1,20 @@
+import { getClient } from '../../common/elasticsearch_client';
+
+export default function metadataHandler(request, response) {
+  /**
+   * @type {ElastalertServer}
+   */
+  
+  var client = getClient();
+
+  client.indices.getMapping({
+    index: request.params.index
+  }).then(function(resp) {
+    response.send(resp);
+  }, function(err) {
+    response.send({
+      error: err
+    });
+  });
+
+}

src/handlers/metadata/get.js

@@ -0,0 +1,42 @@
+import config from '../../common/config';
+import { getClient } from '../../common/elasticsearch_client';
+
+
+function getQueryString(request) {
+  if (request.params.type === 'elastalert_error') {
+    return '*:*';
+  }
+  else {
+    return `rule_name:${request.query.rule_name || '*'}`;
+  }
+}
+
+export default function metadataHandler(request, response) {
+  /**
+   * @type {ElastalertServer}
+   */
+  var client = getClient();
+
+  client.search({
+    index: config.get('writeback_index'),
+    type: request.params.type,
+    body: {
+      from : request.query.from || 0, 
+      size : request.query.size || 100,
+      query: {
+        query_string: {
+          query: getQueryString(request)
+        }
+      },
+      sort: [{ '@timestamp': { order: 'desc' } }]
+    }
+  }).then(function(resp) {
+    resp.hits.hits = resp.hits.hits.map(h => h._source);
+    response.send(resp.hits);
+  }, function(err) {
+    response.send({
+      error: err
+    });
+  });
+
+}

src/handlers/templates/id/post.js

@@ -26,8 +26,8 @@ export default function templatePostHandler(request, response) {
         });
     })
     .catch(function (error) {
-      if (error.error === 'ruleNotFound') {
-        server.templatesController.createRule(request.params.id, body)
+      if (error.error === 'templateNotFound') {
+        server.templatesController.createTemplate(request.params.id, body)
           .then(function () {
             logger.sendSuccessful();
             response.send({