Added listener to exit process when db connection fails

configs/.env

@@ -10,6 +10,7 @@ CONTAINER_HARD_MEMORY_LIMIT_BYTES=2048000000
 CONTAINER_SOFT_MEMORY_LIMIT_BYTES=128000000
 CONTAINER_STOP_LIMIT=10
 COOKIE_SECRET=\$up3r,$3<r3t
+DB_CONNECTION_TIMEOUT=10000
 DEFAULT_PAGE_LIMIT=25
 DOCKER_BUILD_LINE_TIMEOUT_MS=1800000
 DOCKER_BUILD_LOG_TAIL_LIMIT=1000000

configs/.env.test

@@ -9,6 +9,7 @@ BUNYAN_LOG_USE_SRC=true
 CONTAINER_STOP_LIMIT=10
 COOKIE_SECRET=\$up3r,$3<r3t
 CREAM_HOST=localhost:9879
+DB_CONNECTION_TIMEOUT=60000
 DEFAULT_PAGE_LIMIT=25
 DOMAIN=runnable.io
 FULL_API_DOMAIN=http://localhost:3031

lib/models/mongo/mongoose-control.js

@@ -52,6 +52,14 @@ mongooseControl.start = function (cb) {
   }
 
   mongoose.connect(process.env.MONGO, mongooseOptions, cb)
+
+  mongoose.connection.on('disconnected', function () {
+    if (!mongoose.connection._hasOpened) {
+      mongooseControl._exitIfFailedToOpen()
+    } else {
+      mongooseControl._exitIfFailedToReconnect()
+    }
+  })
 }
 
 mongooseControl.stop = function (cb) {
@@ -63,3 +71,18 @@ mongooseControl.stop = function (cb) {
     })
   })
 }
+
+mongooseControl._exitIfFailedToOpen = function () {
+  log.fatal({message: 'Failed to connect to ' + process.env.MONGO + ' failed to establish a connection to mongodb'})
+  process.exit(1)
+}
+
+mongooseControl._exitIfFailedToReconnect = function () {
+  log.error({message: 'Failed to connect to ' + process.env.MONGO + ' failed to establish a connection to mongodb'})
+  setTimeout(function () {
+    if (!mongoose.connection.readyState) {
+      log.fatal({message: 'Exiting nodejs process due to mongodb connection failure'})
+      process.exit(1)
+    }
+  }, process.env.DB_CONNECTION_TIMEOUT)
+}

unit/models/mongo/mongoose-control.js

@@ -11,6 +11,7 @@ var Code = require('code')
 var fs = require('fs')
 var mongoose = require('mongoose')
 var sinon = require('sinon')
+var clock
 
 var mongooseControl = require('models/mongo/mongoose-control')
 
@@ -103,5 +104,73 @@ describe('mongoose-control', function () {
         done()
       })
     })
+
+    describe('handling mongodb disconnect events', function () {
+      beforeEach(function (done) {
+        sinon.stub(mongoose.connection, 'on').yields()
+        sinon.stub(process, 'exit')
+        sinon.stub(mongooseControl, '_exitIfFailedToReconnect')
+        sinon.stub(mongooseControl, '_exitIfFailedToOpen')
+        done()
+      })
+
+      afterEach(function (done) {
+        mongooseControl._exitIfFailedToReconnect.restore()
+        mongooseControl._exitIfFailedToOpen.restore()
+        mongoose.connection.on.restore()
+        process.exit.restore()
+        done()
+      })
+
+      it('should exit if it cannot connect', function (done) {
+        mongooseControl.start(function (err) {
+          expect(err).to.not.exist()
+          sinon.assert.notCalled(mongooseControl._exitIfFailedToReconnect)
+          sinon.assert.calledOnce(mongooseControl._exitIfFailedToOpen)
+          done()
+        })
+      })
+
+      it('should attempt a retry if connection existed', function (done) {
+        mongoose.connection._hasOpened = true
+        mongooseControl.start(function (err) {
+          expect(err).to.not.exist()
+          sinon.assert.notCalled(mongooseControl._exitIfFailedToOpen)
+          sinon.assert.calledOnce(mongooseControl._exitIfFailedToReconnect)
+          done()
+        })
+      })
+    })
+
+    describe('exiting node process when db disconnects', function () {
+      beforeEach(function (done) {
+        clock = sinon.useFakeTimers()
+        sinon.stub(mongoose.connection, 'on').yields()
+        sinon.stub(process, 'exit')
+        done()
+      })
+
+      afterEach(function (done) {
+        mongoose.connection.on.restore()
+        process.exit.restore()
+        clock.restore()
+        done()
+      })
+
+      it('should exit immediately if it cannot connect', function (done) {
+        mongooseControl._exitIfFailedToOpen()
+        sinon.assert.calledOnce(process.exit)
+        done()
+      })
+
+      it('should attempt to reconnect when it was connected once', function (done) {
+        mongooseControl._exitIfFailedToReconnect()
+        clock.tick(1000)
+        sinon.assert.notCalled(process.exit)
+        clock.tick(60000)
+        sinon.assert.calledOnce(process.exit)
+        done()
+      })
+    })
   })
 })