Skip to content

Commit 83a2826

Browse files
bajtosbajtos
authored
Merge pull request #1023 from strongloop/kv-memory
KeyValue access object + memory connector
2 parents e3b6b78 + f15b4e2 commit 83a2826

File tree

12 files changed

+462
-0
lines changed

12 files changed

+462
-0
lines changed

‎index.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,3 +22,5 @@ Object.defineProperty(exports, 'test', {
2222
});
2323

2424
exports.Transaction = require('loopback-connector').Transaction;
25+
26+
exports.KeyValueAccessObject = require('./lib/kvao');

‎lib/connectors/kv-memory.js‎

Lines changed: 143 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,143 @@
1+
'use strict';
2+
3+
var assert = require('assert');
4+
var Connector = require('loopback-connector').Connector;
5+
var debug = require('debug')('loopback:connector:kv-memory');
6+
var util = require('util');
7+
8+
exports.initialize = function initializeDataSource(dataSource, cb) {
9+
var settings = dataSource.settings;
10+
dataSource.connector = new KeyValueMemoryConnector(settings, dataSource);
11+
if (cb) process.nextTick(cb);
12+
};
13+
14+
function KeyValueMemoryConnector(settings, dataSource) {
15+
Connector.call(this, 'kv-memory', settings);
16+
17+
debug('Connector settings', settings);
18+
19+
this.dataSource = dataSource;
20+
this.DataAccessObject = dataSource.juggler.KeyValueAccessObject;
21+
22+
this._store = Object.create(null);
23+
24+
this._setupRegularCleanup();
25+
};
26+
util.inherits(KeyValueMemoryConnector, Connector);
27+
28+
KeyValueMemoryConnector.prototype._setupRegularCleanup = function() {
29+
// Scan the database for expired keys at a regular interval
30+
// in order to release memory. Note that GET operation checks
31+
// key expiration too, the scheduled cleanup is merely a performance
32+
// optimization.
33+
var self = this;
34+
this._cleanupTimer = setInterval(
35+
function() { self._removeExpiredItems(); },
36+
1000);
37+
this._cleanupTimer.unref();
38+
};
39+
40+
KeyValueMemoryConnector._removeExpiredItems = function() {
41+
debug('Running scheduled cleanup of expired items.');
42+
for (var modelName in this._store) {
43+
var modelStore = this._store[modelName];
44+
for (var key in modelStore) {
45+
if (modelStore[key].isExpired()) {
46+
debug('Removing expired key', key);
47+
delete modelStore[key];
48+
}
49+
}
50+
}
51+
};
52+
53+
KeyValueMemoryConnector.prototype._getStoreForModel = function(modelName) {
54+
if (!(modelName in this._store)) {
55+
this._store[modelName] = Object.create(null);
56+
}
57+
return this._store[modelName];
58+
};
59+
60+
KeyValueMemoryConnector.prototype.get =
61+
function(modelName, key, options, callback) {
62+
var store = this._getStoreForModel(modelName);
63+
var item = store[key];
64+
65+
if (item && item.isExpired()) {
66+
debug('Removing expired key', key);
67+
delete store[key];
68+
item = undefined;
69+
}
70+
71+
var value = item ? item.value : null;
72+
73+
debug('GET %j %j -> %s', modelName, key, value);
74+
75+
if (/^buffer:/.test(value)) {
76+
value = new Buffer(value.slice(7), 'base64');
77+
} else if (/^date:/.test(value)) {
78+
value = new Date(value.slice(5));
79+
} else if (value != null) {
80+
value = JSON.parse(value);
81+
}
82+
83+
process.nextTick(function() {
84+
callback(null, value);
85+
});
86+
};
87+
88+
KeyValueMemoryConnector.prototype.set =
89+
function(modelName, key, value, options, callback) {
90+
var store = this._getStoreForModel(modelName);
91+
var value;
92+
if (Buffer.isBuffer(value)) {
93+
value = 'buffer:' + value.toString('base64');
94+
} else if (value instanceof Date) {
95+
value = 'date:' + value.toISOString();
96+
} else {
97+
value = JSON.stringify(value);
98+
}
99+
100+
debug('SET %j %j %s %j', modelName, key, value, options);
101+
store[key] = new StoreItem(value, options && options.ttl);
102+
103+
process.nextTick(callback);
104+
};
105+
106+
KeyValueMemoryConnector.prototype.expire =
107+
function(modelName, key, ttl, options, callback) {
108+
var store = this._getStoreForModel(modelName);
109+
110+
if (!(key in store)) {
111+
return process.nextTick(function() {
112+
callback(new Error('Cannot expire unknown key ' + key));
113+
});
114+
}
115+
116+
debug('EXPIRE %j %j %s', modelName, key, ttl || '(never)');
117+
store[key].setTtl(ttl);
118+
process.nextTick(callback);
119+
};
120+
121+
KeyValueMemoryConnector.prototype.disconnect = function(callback) {
122+
if (this._cleanupTimer)
123+
clearInterval(this._cleanupTimer);
124+
this._cleanupTimer = null;
125+
process.nextTick(callback);
126+
};
127+
128+
function StoreItem(value, ttl) {
129+
this.value = value;
130+
this.setTtl(ttl);
131+
}
132+
133+
StoreItem.prototype.isExpired = function() {
134+
return this.expires && this.expires <= Date.now();
135+
};
136+
137+
StoreItem.prototype.setTtl = function(ttl) {
138+
if (ttl) {
139+
this.expires = Date.now() + ttl;
140+
} else {
141+
this.expires = undefined;
142+
}
143+
};

‎lib/datasource.js‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@ var assert = require('assert');
2121
var async = require('async');
2222
var traverse = require('traverse');
2323
var g = require('strong-globalize')();
24+
var juggler = require('..');
2425

2526
if (process.env.DEBUG === 'loopback') {
2627
// For back-compatibility
@@ -107,6 +108,7 @@ function DataSource(name, settings, modelBuilder) {
107108
this.modelBuilder = modelBuilder || new ModelBuilder();
108109
this.models = this.modelBuilder.models;
109110
this.definitions = this.modelBuilder.definitions;
111+
this.juggler = juggler;
110112

111113
// operation metadata
112114
// Initialize it before calling setup as the connector might register operations

‎lib/kvao/expire.js‎

Lines changed: 33 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,33 @@
1+
'use strict';
2+
3+
var assert = require('assert');
4+
var utils = require('../utils');
5+
6+
/**
7+
* Set expiration (TTL) for the given key.
8+
*
9+
* @param {String} key
10+
* @param {Number} ttl
11+
* @param {Object} options
12+
* @callback cb
13+
* @param {Error} error
14+
*
15+
* @header KVAO.get(key, cb)
16+
*/
17+
module.exports = function keyValueExpire(key, ttl, options, callback) {
18+
if (callback == undefined && typeof options === 'function') {
19+
callback = options;
20+
options = {};
21+
} else if (!options) {
22+
options = {};
23+
}
24+
25+
assert(typeof key === 'string' && key, 'key must be a non-empty string');
26+
assert(typeof ttl === 'number' && ttl > 0, 'ttl must be a positive integer');
27+
assert(typeof options === 'object', 'options must be an object');
28+
29+
callback = callback || utils.createPromiseCallback();
30+
this.getConnector().expire(this.modelName, key, ttl, options, callback);
31+
return callback.promise;
32+
};
33+

‎lib/kvao/get.js‎

Lines changed: 32 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,32 @@
1+
'use strict';
2+
3+
var assert = require('assert');
4+
var utils = require('../utils');
5+
6+
/**
7+
* Get the value stored for the given key.
8+
*
9+
* @param {String} key
10+
* @callback cb
11+
* @param {Error} error
12+
* @param {*} value
13+
*
14+
* @header KVAO.get(key, cb)
15+
*/
16+
module.exports = function keyValueGet(key, options, callback) {
17+
if (callback == undefined && typeof options === 'function') {
18+
callback = options;
19+
options = {};
20+
} else if (!options) {
21+
options = {};
22+
}
23+
24+
assert(typeof key === 'string' && key, 'key must be a non-empty string');
25+
26+
callback = callback || utils.createPromiseCallback();
27+
this.getConnector().get(this.modelName, key, options, function(err, result) {
28+
// TODO convert raw result to Model instance (?)
29+
callback(err, result);
30+
});
31+
return callback.promise;
32+
};

‎lib/kvao/index.js‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
'use strict';
2+
3+
function KeyValueAccessObject() {
4+
};
5+
6+
module.exports = KeyValueAccessObject;
7+
8+
KeyValueAccessObject.get = require('./get');
9+
KeyValueAccessObject.set = require('./set');
10+
KeyValueAccessObject.expire = require('./expire');
11+
12+
KeyValueAccessObject.getConnector = function() {
13+
return this.getDataSource().connector;
14+
};
15+

‎lib/kvao/set.js‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,40 @@
1+
'use strict';
2+
3+
var assert = require('assert');
4+
var utils = require('../utils');
5+
6+
/**
7+
* Set the value for the given key.
8+
*
9+
* @param {String} key
10+
* @param {*} value
11+
* @callback cb
12+
* @param {Error} error
13+
*
14+
* @header KVAO.set(key, value, cb)
15+
*/
16+
module.exports = function keyValueSet(key, value, options, callback) {
17+
if (callback == undefined && typeof options === 'function') {
18+
callback = options;
19+
options = {};
20+
} else if (typeof options === 'number') {
21+
options = { ttl: options };
22+
} else if (!options) {
23+
options = {};
24+
}
25+
26+
assert(typeof key === 'string' && key, 'key must be a non-empty string');
27+
assert(value != null, 'value must be defined and not null');
28+
assert(typeof options === 'object', 'options must be an object');
29+
if (options && 'ttl' in options) {
30+
assert(typeof options.ttl === 'number' && options.ttl > 0,
31+
'options.ttl must be a positive number');
32+
}
33+
34+
callback = callback || utils.createPromiseCallback();
35+
36+
// TODO convert possible model instance in "value" to raw data via toObect()
37+
this.getConnector().set(this.modelName, key, value, options, callback);
38+
return callback.promise;
39+
};
40+

‎test/kv-memory.js‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
var kvMemory = require('../lib/connectors/kv-memory');
2+
var DataSource = require('..').DataSource;
3+
4+
describe('KeyValue-Memory connector', function() {
5+
var lastDataSource;
6+
var dataSourceFactory = function() {
7+
lastDataSource = new DataSource({ connector: kvMemory });
8+
return lastDataSource;
9+
};
10+
11+
afterEach(function disconnectKVMemoryConnector() {
12+
if (lastDataSource) return lastDataSource.disconnect();
13+
});
14+
15+
require('./kvao.suite')(dataSourceFactory);
16+
});

‎test/kvao.suite.js‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
'use strict';
2+
3+
var debug = require('debug')('test');
4+
var fs = require('fs');
5+
var path = require('path');
6+
7+
module.exports = function(dataSourceFactory, connectorCapabilities) {
8+
describe('KeyValue API', function loadAllTestFiles() {
9+
var testRoot = path.resolve(__dirname, 'kvao');
10+
var testFiles = fs.readdirSync(testRoot);
11+
testFiles = testFiles.filter(function(it) {
12+
return !!require.extensions[path.extname(it).toLowerCase()] &&
13+
/\.suite\.[^.]+$/.test(it);
14+
});
15+
16+
for (var ix in testFiles) {
17+
var name = testFiles[ix];
18+
var fullPath = path.resolve(testRoot, name);
19+
debug('Loading test suite %s (%s)', name, fullPath);
20+
require(fullPath)(dataSourceFactory, connectorCapabilities);
21+
}
22+
});
23+
};

‎test/kvao/_helpers.js‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,9 @@
1+
'use strict';
2+
3+
exports.givenCacheItem = function(dataSourceFactory) {
4+
var dataSource = dataSourceFactory();
5+
return dataSource.createModel('CacheItem', {
6+
key: String,
7+
value: 'any',
8+
});
9+
};

0 commit comments

Comments
 (0)