Bulkmode support

.travis.yml

@@ -8,15 +8,9 @@ node_js:
 before_install:
   - travis_retry npm install -g [email protected]
   - travis_retry npm install --loglevel=http
-  - >
-    openssl aes-256-cbc \
-      -K $encrypted_4fb8e306b739_key \
-      -iv $encrypted_4fb8e306b739_iv \
-      -in test/config.json.enc \
-      -out test/config.json -d
 
 script:
-  - npm test
+  - npm run test-as-mock
 
 matrix:
   allow_failures:

README.md

@@ -158,11 +158,11 @@ See the [Loggly search guide][search] for more details on how to effectively sea
 ```
 
 ## Run Tests
-All of the node-loggly tests are written in [vows][vows], and cover all of the use cases described above. You will need to add your Loggly username, password, subdomain, and a two test inputs to test/data/test-config.json before running tests. When configuring the test inputs on Loggly, the first test input should be named 'test' using the HTTP service. The second input should be name 'test_json' using the HTTP service with the JSON logging option enabled:
+All of the node-loggly tests are written in [vows][vows], and cover all of the use cases described above. You will need to add your Loggly username, password, subdomain, and your loggly token to test/config.json before running tests.
 
 ``` js
   {
-    "token": "your-really-long-token-you-got-when-you-created-an-http-input",
+    "token": "your-loggly-token",
     "subdomain": "your-subdomain",
     "auth": {
       "username": "your-username",

lib/loggly/client.js

@@ -46,6 +46,7 @@ var Loggly = exports.Loggly = function (options) {
   }
 
   events.EventEmitter.call(this);
+
   this.subdomain    = options.subdomain;
   this.token        = options.token;
   this.host         = options.host || 'logs-01.loggly.com';
@@ -54,7 +55,7 @@ var Loggly = exports.Loggly = function (options) {
   this.proxy        = options.proxy || null;
   this.userAgent    = 'node-loggly ' + loggly.version;
   this.useTagHeader = 'useTagHeader' in options ? options.useTagHeader : true;
-
+  this.isBulk       = options.isBulk || false;
   //
   // Set the tags on this instance.
   //
@@ -101,7 +102,6 @@ Loggly.prototype.log = function (msg, tags, callback) {
   // Remark: Have some extra logic for detecting if we want to make a bulk
   // request to loggly
   //
-  var isBulk = Array.isArray(msg);
   function serialize(msg) {
     if (msg instanceof Object) {
       return self.json ? stringify(msg) : common.serialize(msg);
@@ -111,13 +111,14 @@ Loggly.prototype.log = function (msg, tags, callback) {
     }
   }
 
-  msg = isBulk ? msg.map(serialize).join('\n') : serialize(msg);
+  msg = serialize(msg);
 
   logOptions = {
-    uri:     isBulk ? this.urls.bulk : this.urls.log,
+    uri:     this.isBulk ? this.urls.bulk : this.urls.log,
     method:  'POST',
     body:    msg,
     proxy:   this.proxy,
+    isBulk: this.isBulk,
     headers: {
       host:             this.host,
       accept:           '*/*',

lib/loggly/common.js

@@ -6,6 +6,13 @@
  *
  */
 
+//
+// Variables for Bulk
+//
+var arrSize = 100,
+    arrMsg = [],
+    timerFunction = null;
+
 var https = require('https'),
     util = require('util'),
     request = require('request'),
@@ -69,6 +76,7 @@ common.loggly = function () {
       method,
       auth,
       proxy,
+      isBulk,
       uri;
 
   //
@@ -88,6 +96,7 @@ common.loggly = function () {
       uri         = args[0].uri;
       requestBody = args[0].body;
       auth        = args[0].auth;
+      isBulk      = args[0].isBulk;
       headers     = args[0].headers;
       proxy       = args[0].proxy;
     }
@@ -109,41 +118,81 @@ common.loggly = function () {
       if (callback) { callback(err) }
     }
   }
-
   var requestOptions = {
-    uri: uri,
+    uri: isBulk ? uri + '/tag/' + headers['X-LOGGLY-TAG'] : uri,
     method: method,
-    headers: headers || {},
+    headers: isBulk ? {} : headers || {},             // Set headers empty for bulk
     proxy: proxy
   };
 
   if (auth) {
     requestOptions.headers.authorization = 'Basic ' + new Buffer(auth.username + ':' + auth.password).toString('base64');
   }
-
   if (requestBody) {
     requestOptions.body = requestBody;
   }
-
-  try {
-    request(requestOptions, function (err, res, body) {
-      if (err) {
-        return onError(err);
-      }
-
-      var statusCode = res.statusCode.toString();
-      if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
-        return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
-      }
-
-      success(res, body);
-    });
+  function sendLogs() {
+    try {
+      request(requestOptions, function (err, res, body) {
+        if (err) {
+          return onError(err);
+        }
+        var statusCode = res.statusCode.toString();
+        if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
+          return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+        }
+        success(res, body);
+      });
+    }
+    catch (ex) {
+      onError(ex);
+    }
   }
-  catch (ex) {
-    onError(ex);
+  function sendBulkLogs() {
+    if (arrMsg.length === 0) {
+      return;
+    }
+    //
+    // Join Array Message with new line ('\n') character
+    //
+    requestOptions.body = arrMsg.join('\n');
+    try {
+      request(requestOptions, function (err, res, body) {
+        if (err) {
+          return onError(err);
+        }
+        var statusCode = res.statusCode.toString();
+        if (Object.keys(failCodes).indexOf(statusCode) !== -1) {
+          return onError((new Error('Loggly Error (' + statusCode + '): ' + failCodes[statusCode])));
+        }
+        success(res, body);
+      });
+    }
+    catch (ex) {
+      onError(ex);
+    }
+    finally {
+      //
+      // Empty the array
+      //
+      arrMsg.length = 0;
+    }
+  }
+  if (isBulk === true) {
+    if (timerFunction === null) {
+      timerFunction = setInterval(function () {
+        sendBulkLogs();
+      },30000);
+    }
+    arrMsg.push(requestBody);
+    if (arrMsg.length === arrSize) {
+      sendBulkLogs();
+    }
+  }
+  else {
+    sendLogs();
   }
 };
-
 //
 // ### function serialize (obj, key)
 // #### @obj {Object|literal} Object to serialize

package.json

@@ -16,16 +16,20 @@
   "dependencies": {
     "request": "2.67.x",
     "timespan": "2.3.x",
-    "json-stringify-safe": "5.0.x"
+    "json-stringify-safe": "5.0.x",
+    "nock": "~7.2.2"
   },
   "devDependencies": {
     "common-style": "^3.1.0",
-    "vows": "0.8.x"
+    "vows": "0.8.x",
+    "nock": "~7.2.2"
   },
   "main": "./lib/loggly",
   "scripts": {
     "pretest": "common lib/**/*.js lib/*.js test/helpers.js",
-    "test": "vows test/*-test.js --spec"
+    "test": "vows test/*-test.js --spec",
+    "pretest-as-mock": "common lib/**/*.js lib/*.js test-as-mock/helpers.js",
+    "test-as-mock": "vows test-as-mock/*-test.js --spec"
   },
   "license": "MIT",
   "engines": {

test-as-mock/common-test.js

@@ -0,0 +1,32 @@
+/*
+ * common-test.js: Tests for Loggly `common` utility module
+ *
+ * (C) 2010 Charlie Robbins
+ * MIT LICENSE
+ *
+ */
+
+var path = require('path'),
+    vows = require('vows'),
+    assert = require('assert'),
+    common = require('../lib/loggly/common');
+
+vows.describe('node-loggly/common').addBatch({
+  "When using the common module": {
+    "the clone() method": {
+      topic: function () {
+        this.obj = {
+          name: 'common',
+          deep: {
+            first: 'first',
+            second: 'second'
+          }
+        };
+        return common.clone(this.obj);
+      },
+      "should return a deep clone of the object": function (clone) {
+        assert.isFalse(this.obj.deep === clone.deep);
+      }
+    }
+  }
+}).export(module);

test-as-mock/config.json

@@ -0,0 +1,8 @@
+{
+    "token": "test",
+    "subdomain": "testSubdomain",
+    "auth": {
+      "username": "test",
+      "password": "test"
+    }
+  }

test-as-mock/customer-test.js

@@ -0,0 +1,43 @@
+/*
+ * search-test.js: Tests for Loggly search requests
+ *
+ * (C) 2010 Charlie Robbins
+ * MIT LICENSE
+ *
+ */
+var path = require('path'),
+  vows = require('vows'),
+  assert = require('assert'),
+  nock = require('nock'),
+  helpers = require('./helpers');
+
+var options = {},
+  testContext = {},
+  config = helpers.loadConfig(),
+  loggly = require('../lib/loggly').createClient(config);
+
+vows.describe('node-loggly/customer').addBatch({
+  "When using the node-loggly client": {
+    "the customer() method": {
+      topic: function() {
+        nock("https://" + config.subdomain + ".loggly.com")
+          .get('/apiv2/customer')
+          .reply(200, {
+            "tokens": ["test", "test2"],
+            "subdomain": config.subdomain,
+            "subscription": {
+              "key1": "value1"
+            }
+          });
+        loggly.customer(this.callback);
+
+      },
+      "should return a valid customer": function(err, customer) {
+        assert.isNull(err);
+        assert.isArray(customer.tokens);
+        assert.isString(customer.subdomain);
+        assert.isObject(customer.subscription);
+      }
+    }
+  }
+}).export(module);

test-as-mock/helpers.js

@@ -0,0 +1,48 @@
+/*
+ * helpers.js: Test helpers for node-loggly
+ *
+ * (C) 2010 Charlie Robbins
+ * MIT LICENSE
+ *
+ */
+
+var fs = require('fs'),
+    util = require('util'),
+    path = require('path'),
+    vows = require('vows'),
+    assert = require('assert'),
+    loggly = require('../lib/loggly');
+
+var helpers = exports;
+
+helpers.validConfig = function (config) {
+  return config
+    && config.subdomain !== 'test-subdomain'
+    && config.auth
+    && config.auth.username !== 'test-username'
+    && config.auth.password !== 'test-password'
+    && config.token;
+};
+
+helpers.loadConfig = function () {
+  try {
+    var config = require('./config');
+    if (!helpers.validConfig(config)) {
+      throw new Error(util.format('test/config.json: invalid data %j', config));
+    }
+
+    helpers.config = config || {};
+    return helpers.config;
+  }
+  catch (ex) {
+    console.log('Error parsing test/config.json');
+    throw ex;
+  }
+};
+
+helpers.assertSearch = function (err, results) {
+  assert.isNull(err);
+  assert.isObject(results);
+  assert.isArray(results.events);
+  assert.isTrue(typeof results.total_events !== 'undefined');
+};