From 40bdde2f858232b924514277e14b0efe2ca482ae Mon Sep 17 00:00:00 2001 From: Daniel Ennis Date: Thu, 12 Oct 2017 16:03:42 -0400 Subject: [PATCH] feat(pool): Add Minimum connections, and auto close extra connections This adds a new configuration for desired minimum connections, to complement the connection limit. If the autoOpenConnections is enabled (default true), the pool will immediately open the desired minimum number of connections. Any connection opened above the minimum, who sits unused for a time specified in idleTimeout, will be automatically closed. This allows you to set up a minimum expectation of standard load for connections to your database, but also allow you to configure a higher maximum to handle unexpected burst traffic, and be able to close them automatically when the burst subsides. --- lib/connection.js | 3 +- lib/pool.js | 77 ++++++++++++++++++++++++++++++++++++++++++---- lib/pool_config.js | 20 +++++++++--- 3 files changed, 88 insertions(+), 12 deletions(-) diff --git a/lib/connection.js b/lib/connection.js index 69ad5cbcd3..c4e60b4cd0 100644 --- a/lib/connection.js +++ b/lib/connection.js @@ -103,6 +103,7 @@ function Connection(opts) { }); this.stream.on('end', function() { + this._closed = true; // we need to set this flag everywhere where we want connection to close if (connection._closing) { return; @@ -946,8 +947,8 @@ Connection.prototype.serverHandshake = function serverHandshake(args) { Connection.prototype.end = function(callback) { var connection = this; + connection._closing = true; if (this.config.isServer) { - connection._closing = true; var quitCmd = new EventEmitter(); setImmediate(function() { connection.stream.end(); diff --git a/lib/pool.js b/lib/pool.js index d80402d710..93dd332ad5 100644 --- a/lib/pool.js +++ b/lib/pool.js @@ -1,6 +1,7 @@ var mysql = require('../index.js'); var EventEmitter = require('events').EventEmitter; +var Timers = require('timers'); var Util = require('util'); var PoolConnection = require('./pool_connection.js'); var Queue = require('denque'); @@ -18,6 +19,24 @@ function Pool(options) { this._freeConnections = new Queue(); this._connectionQueue = new Queue(); this._closed = false; + if (this.config.autoOpenConnections && this.config.minConnections) { + var self = this; + var opened = 0; + for (var i = 0; i < this.config.minConnections; i++) { + this.getConnection(function (err, conn) { + if (conn) { + process.nextTick(function () { + if (++opened === self.config.minConnections) { + self._openingConnections = false; + } + self.releaseConnection(conn); + }); + } + }); + } + // This must be after the getConnection, or else its enqueued + this._openingConnections = true; + } } Pool.prototype.getConnection = function(cb) { @@ -26,6 +45,10 @@ Pool.prototype.getConnection = function(cb) { return cb(new Error('Pool is closed.')); }); } + if (this._openingConnections) { + // We are opening a connect, use it when ready + return this._connectionQueue.push(cb); + } var connection; @@ -82,7 +105,8 @@ Pool.prototype.getConnection = function(cb) { Pool.prototype.releaseConnection = function(connection) { var cb; - if (!connection._pool) { + connection._lastReleased = Date.now(); + if (!connection._pool || connection._closed || connection._closing) { // The connection has been removed from the pool and is no longer good. if (this._connectionQueue.length) { cb = this._connectionQueue.shift(); @@ -95,6 +119,7 @@ Pool.prototype.releaseConnection = function(connection) { process.nextTick(cb.bind(null, null, connection)); } else { this._freeConnections.push(connection); + this._manageExpiredTimer(); } }; @@ -111,14 +136,14 @@ Pool.prototype.end = function(cb) { var calledBack = false; var closedConnections = 0; - var connection; + var numConnections = this._allConnections.length; var endCB = function(err) { if (calledBack) { return; } - if (err || ++closedConnections >= this._allConnections.length) { + if (err || ++closedConnections >= numConnections) { calledBack = true; cb(err); return; @@ -130,9 +155,9 @@ Pool.prototype.end = function(cb) { return; } - for (var i = 0; i < this._allConnections.length; i++) { - connection = this._allConnections.get(i); - connection._realEnd(endCB); + var connection; + while ((connection = this._allConnections.shift())) { + this._closeConnection(connection, endCB); } }; @@ -184,6 +209,39 @@ Pool.prototype.execute = function(sql, values, cb) { }); }; +Pool.prototype._manageExpiredTimer = function() { + var hasExtra = this._allConnections.length > this.config.minConnections; + if (hasExtra && !this._expiredTimer) { + this._expiredTimer = Timers.setInterval( + Pool.prototype._closeIdleConnections.bind(this), + Math.min(15, this.config.idleTimeout) * 1000 + ); + } else if (!hasExtra && this._expiredTimer) { + Timers.clearInterval(this._expiredTimer); + this._expiredTimer = null; + } +}; + +Pool.prototype._closeIdleConnections = function() { + var now = Date.now(); + var timeout = this.config.idleTimeout * 1000; + var length = this._freeConnections.length; + var numExtra = this._allConnections.length - this.config.minConnections; + for (var i = 0; numExtra > 0 && i < length; i++) { + var conn = this._freeConnections.get(i); + + if (now > conn._lastReleased + timeout) { + // This connection has been unused for longer than the timeout + this._closeConnection(conn); + // decrement i and length as the length will be reduced by 1 by closeConnection + i--; + length--; + numExtra--; + } + } + this._manageExpiredTimer(); +}; + Pool.prototype._removeConnection = function(connection) { // Remove connection from all connections spliceConnection(this._allConnections, connection); @@ -192,6 +250,13 @@ Pool.prototype._removeConnection = function(connection) { spliceConnection(this._freeConnections, connection); this.releaseConnection(connection); + + this._manageExpiredTimer(); +}; + +Pool.prototype._closeConnection = function (connection, cb) { + connection._realEnd(cb); + this._removeConnection(connection); }; Pool.prototype.format = function(sql, values) { diff --git a/lib/pool_config.js b/lib/pool_config.js index 782d48c578..5ee3cebf9f 100644 --- a/lib/pool_config.js +++ b/lib/pool_config.js @@ -6,13 +6,23 @@ function PoolConfig(options) { options = ConnectionConfig.parseUrl(options); } this.connectionConfig = new ConnectionConfig(options); - this.waitForConnections = options.waitForConnections === undefined + this.waitForConnections = options.waitForConnections == null ? true : Boolean(options.waitForConnections); - this.connectionLimit = options.connectionLimit === undefined + this.connectionLimit = options.connectionLimit == null ? 10 - : Number(options.connectionLimit); - this.queueLimit = options.queueLimit === undefined + : Number(options.connectionLimit) || 10; + this.queueLimit = options.queueLimit == null ? 0 - : Number(options.queueLimit); + : Number(options.queueLimit) || 0; + this.minConnections = Math.min( + this.connectionLimit, + Number(options.minConnections) || 0 + ); + this.autoOpenConnections = options.autoOpenConnections == null + ? true + : Boolean(options.autoOpenConnections); + this.idleTimeout = options.idleTimeout == null + ? 300 + : Number(options.idleTimeout) || 300; }