Merge branch 'nokia-artif_api' into develop

Author: martijnrondeel

README.md

@@ -185,7 +185,18 @@ This server exposes the following REST API's:
 - **[WIP] POST `/config`**
 
     Allows you to edit the ElastAlert configuration from `config.yaml` in `elastalertPath` (from the config). The required body to be send will be edited when the work on this API is done.
- 
+
+- **[WIP] POST `/download`**
+  
+    Allows you to download a .tar archive with rules from a given HTTP endpoint. The archive will be downloaded, extracted and removed.
+    Please note, body should contain URL pointing to tar archive, with tar extension.	
+   
+    Usage example:
+	
+    ```bash
+    curl -X POST localhost:3030/download -d "url=https://artifactory.com:443/artifactory/raw/rules/rules.tar"
+    ```
+        
 ## Contributing
 Want to contribute to this project? Great! Please read our [contributing guidelines](CONTRIBUTING.md) before submitting an issue or a pull request.
 

package.json

@@ -29,7 +29,10 @@
     "mkdirp": "^0.5.1",
     "object-resolve-path": "^1.1.1",
     "randomstring": "^1.1.5",
-    "raven": "^2.3.0"
+    "raven": "^2.3.0",
+    "tar": "^4.4.1",
+    "fs-extra": "^5.0.0",
+    "request-promise-native": "^1.0.5"
   },
   "devDependencies": {
     "eslint": "^4.17.0",

src/common/errors/rule_request_errors.js

@@ -41,3 +41,13 @@ export class RulesRootFolderNotCreatableError extends RequestError {
     super('rulesRootFolderNotCreatable', 'The rules folder wasn\'t found and couldn\'t be created by the file system.', 403);
   }
 }
+export class URLNotSentError extends RequestError {
+  constructor() {
+    super('URLNotSentError', 'URL is missing in the body', 400);
+  }
+}
+export class URLNotPointingTar extends RequestError {
+  constructor() {
+    super('URLNotPointingTar', 'URL is not pointing to .tar file', 400);
+  }
+}

src/controllers/rules/index.js

@@ -1,10 +1,16 @@
-import {join as joinPath, normalize as normalizePath, extname as pathExtension} from 'path';
+import { join as joinPath, normalize as normalizePath, extname as pathExtension } from 'path';
 import mkdirp from 'mkdirp';
+import tar from 'tar';
+import fs from 'fs-extra';
+import rq from 'request-promise-native';
 import FileSystem from '../../common/file_system';
 import config from '../../common/config';
 import Logger from '../../common/logger';
-import {RuleNotFoundError, RuleNotReadableError, RuleNotWritableError,
-  RulesFolderNotFoundError, RulesRootFolderNotCreatableError} from '../../common/errors/rule_request_errors';
+import path from 'path';
+import {
+  RuleNotFoundError, RuleNotReadableError, RuleNotWritableError,
+  RulesFolderNotFoundError, RulesRootFolderNotCreatableError
+} from '../../common/errors/rule_request_errors';
 
 let logger = new Logger('RulesController');
 
@@ -17,26 +23,26 @@ export default class RulesController {
   getRules(path) {
     const self = this;
     const fullPath = joinPath(self.rulesFolder, path);
-    return new Promise(function (resolve, reject) {
+    return new Promise(function(resolve, reject) {
       self._fileSystemController.readDirectory(fullPath)
-        .then(function (directoryIndex) {
+        .then(function(directoryIndex) {
 
-          directoryIndex.rules = directoryIndex.files.filter(function (fileName) {
+          directoryIndex.rules = directoryIndex.files.filter(function(fileName) {
             return pathExtension(fileName).toLowerCase() === '.yaml';
-          }).map(function (fileName) {
+          }).map(function(fileName) {
             return fileName.slice(0, -5);
           });
 
           delete directoryIndex.files;
           resolve(directoryIndex);
         })
-        .catch(function (error) {
+        .catch(function(error) {
 
           // Check if the requested folder is the rules root folder
           if (normalizePath(self.rulesFolder) === fullPath) {
 
             // Try to create the root folder
-            mkdirp(fullPath, function (error) {
+            mkdirp(fullPath, function(error) {
               if (error) {
                 reject(new RulesRootFolderNotCreatableError());
                 logger.warn(`The rules root folder (${fullPath}) couldn't be found nor could it be created by the file system.`);
@@ -54,29 +60,29 @@ export default class RulesController {
 
   rule(id) {
     const self = this;
-    return new Promise(function (resolve, reject) {
+    return new Promise(function(resolve, reject) {
       self._findRule(id)
-        .then(function (access) {
+        .then(function(access) {
           console.log('rule resolved');
           resolve({
-            get: function () {
+            get: function() {
               if (access.read) {
                 return self._getRule(id);
               }
               return self._getErrorPromise(new RuleNotReadableError(id));
             },
-            edit: function (body) {
+            edit: function(body) {
               if (access.write) {
                 return self._editRule(id, body);
               }
               return self._getErrorPromise(new RuleNotWritableError(id));
             },
-            delete: function () {
+            delete: function() {
               return self._deleteRule(id);
             }
           });
         })
-        .catch(function () {
+        .catch(function() {
           console.log('catched');
           reject(new RuleNotFoundError(id));
         });
@@ -87,12 +93,16 @@ export default class RulesController {
     return this._editRule(id, content);
   }
 
+  downloadRules(URL) {
+    return this._downloadRules(URL);
+  }
+
   _findRule(id) {
     let fileName = id + '.yaml';
     const self = this;
-    return new Promise(function (resolve, reject) {
+    return new Promise(function(resolve, reject) {
       self._fileSystemController.fileExists(joinPath(self.rulesFolder, fileName))
-        .then(function (exists) {
+        .then(function(exists) {
           if (!exists) {
             reject();
           } else {
@@ -104,7 +114,7 @@ export default class RulesController {
             });
           }
         })
-        .catch(function (error) {
+        .catch(function(error) {
           reject(error);
         });
     });
@@ -125,8 +135,21 @@ export default class RulesController {
     return this._fileSystemController.deleteFile(path);
   }
 
+  _downloadRules(URL) {
+    const options = {
+      uri: URL,
+      strictSSL: false
+    };
+    const filename = path.basename(URL);
+
+    return rq.get(options)
+      .then(buffer => fs.outputFile(filename, buffer)
+        .then(() => this._untarFile(this.rulesFolder, filename))
+        .then(() => fs.remove(filename)));
+  }
+
   _getErrorPromise(error) {
-    return new Promise(function (resolve, reject) {
+    return new Promise(function(resolve, reject) {
       reject(error);
     });
   }
@@ -140,4 +163,13 @@ export default class RulesController {
       return ruleFolderSettings.path;
     }
   }
+
+  _untarFile(path_to_extract, archive) {
+    return tar.extract(
+      {
+        cwd: path_to_extract,
+        file: archive
+      }
+    );
+  }
 }

src/handlers/rules/id/download.js

@@ -0,0 +1,50 @@
+import RouteLogger from '../../../routes/route_logger';
+import { sendRequestError } from '../../../common/errors/utils';
+import { URLNotSentError, URLNotPointingTar } from '../../../common/errors/rule_request_errors';
+import path from 'path';
+
+let logger = new RouteLogger('/download');
+
+
+export default function downloadRulesHandler(request, response) {
+  /**
+   * @type {ElastalertServer}
+   */
+  let server = request.app.get('server');
+  let body = request.body;
+
+
+  function _test_body(body) {
+    if (!body || !body.url) {
+      return new URLNotSentError();
+    }
+    let filename = path.basename(body.url);
+    if (!filename.endsWith('.tar')) {
+      return new URLNotPointingTar();
+    }
+    return body;
+  }
+
+  body = _test_body(body);
+  if (body.error) {
+    logger.sendFailed(body.error);
+    sendRequestError(response, body.error);
+    return;
+  }
+
+
+  server.rulesController.downloadRules(body.url)
+    .then(() => {
+      logger.sendSuccessful();
+      return response.send({
+        downloaded: true,
+        extracted: true,
+        url: body.url
+      });
+    })
+    .catch((error) => {
+      logger.sendFailed(error);
+      sendRequestError(response, error);
+    });
+
+}

src/routes/routes.js

@@ -4,6 +4,7 @@ import controlHandler from '../handlers/status/control';
 import errorsHandler from '../handlers/status/errors';
 import rulesHandler from '../handlers/rules';
 import ruleGetHandler from '../handlers/rules/id/get';
+import downloadRulesHandler from '../handlers/rules/id/download';
 import rulePostHandler from '../handlers/rules/id/post';
 import ruleDeleteHandler from '../handlers/rules/id/delete';
 import templatesHandler from '../handlers/templates';
@@ -68,6 +69,11 @@ let routes = [
     path: 'config',
     method: ['GET', 'POST'],
     handler: [configGetHandler, configPostHandler]
+  },
+  {
+    path: 'download',
+    method: ['POST'],
+    handler: [downloadRulesHandler]
   }
 ];