Merge pull request #1049 from strongloop/feature/kvao-iterate-keys

kvao: add iterateKeys() and keys()

Author: bajtos

.npmignore

@@ -9,3 +9,4 @@ benchmark.js
 analyse.r
 docs/html
 npm-debug.log
+.travis.yml

lib/connectors/kv-memory.js

@@ -3,6 +3,7 @@
 var assert = require('assert');
 var Connector = require('loopback-connector').Connector;
 var debug = require('debug')('loopback:connector:kv-memory');
+var minimatch = require('minimatch');
 var util = require('util');
 
 exports.initialize = function initializeDataSource(dataSource, cb) {
@@ -71,7 +72,9 @@ KeyValueMemoryConnector.prototype._removeIfExpired = function(modelName, key) {
     debug('Removing expired key', key);
     delete store[key];
     item = undefined;
+    return true;
   }
+  return false;
 };
 
 KeyValueMemoryConnector.prototype.get =
@@ -154,6 +157,39 @@ function(modelName, key, options, callback) {
   });
 };
 
+KeyValueMemoryConnector.prototype.iterateKeys =
+function(modelName, filter, options, callback) {
+  var store = this._getStoreForModel(modelName);
+  var self = this;
+  var checkFilter = createMatcher(filter.match);
+
+  var keys = Object.keys(store).filter(function(key) {
+    return !self._removeIfExpired(modelName, key) && checkFilter(key);
+  });
+
+  debug('ITERATE KEYS %j -> %s keys', modelName, keys.length);
+
+  var ix = 0;
+  return {
+    next: function(cb) {
+      var value = ix < keys.length ? keys[ix++] : undefined;
+      setImmediate(function() { cb(null, value); });
+    },
+  };
+};
+
+function createMatcher(pattern) {
+  if (!pattern) return function matchAll() { return true; };
+
+  return minimatch.filter(pattern, {
+    nobrace: true,
+    noglobstar: true,
+    dot: true,
+    noext: true,
+    nocomment: true,
+  });
+}
+
 KeyValueMemoryConnector.prototype.disconnect = function(callback) {
   if (this._cleanupTimer)
     clearInterval(this._cleanupTimer);

lib/kvao/index.js

@@ -9,6 +9,8 @@ KeyValueAccessObject.get = require('./get');
 KeyValueAccessObject.set = require('./set');
 KeyValueAccessObject.expire = require('./expire');
 KeyValueAccessObject.ttl = require('./ttl');
+KeyValueAccessObject.iterateKeys = require('./iterate-keys');
+KeyValueAccessObject.keys = require('./keys');
 
 KeyValueAccessObject.getConnector = function() {
   return this.getDataSource().connector;

lib/kvao/iterate-keys.js

@@ -0,0 +1,39 @@
+'use strict';
+
+var assert = require('assert');
+var utils = require('../utils');
+
+/**
+ * Asynchronously iterate all keys.
+ *
+ * @param {Object} filter An optional filter object with the following
+ * properties:
+ *  - `match` - glob string to use to filter returned keys, e.g. 'userid.*'
+ *    All connectors are required to support `*` and `?`.
+ *    They may also support additional special characters that are specific
+ *    to the backing store.
+ *
+ * @param {Object} options
+ *
+ * @returns {AsyncIterator} An object implementing "next(cb) -> Promise"
+ * function that can be used to iterate all keys.
+ *
+ * @header KVAO.iterateKeys(filter)
+ */
+module.exports = function keyValueIterateKeys(filter, options) {
+  filter = filter || {};
+  options = options || {};
+
+  assert(typeof filter === 'object', 'filter must be an object');
+  assert(typeof options === 'object', 'options must be an object');
+
+  var iter = this.getConnector().iterateKeys(this.modelName, filter, options);
+  // promisify the returned iterator
+  return {
+    next: function(callback) {
+      callback = callback || utils.createPromiseCallback();
+      iter.next(callback);
+      return callback.promise;
+    },
+  };
+};

lib/kvao/keys.js

@@ -0,0 +1,60 @@
+'use strict';
+
+var assert = require('assert');
+var utils = require('../utils');
+
+/**
+ * Get all keys.
+ *
+ * **NOTE**
+ * Building an in-memory array of all keys may be expensive.
+ * Consider using `iterateKeys` instead.
+ *
+ * @param {Object} filter An optional filter object with the following
+ * properties:
+ *  - `match` - glob string to use to filter returned keys, e.g. 'userid.*'
+ *    All connectors are required to support `*` and `?`.
+ *    They may also support additional special characters that are specific
+ *    to the backing store.
+ * @param {Object} options
+ * @callback callback
+ * @param {Error=} err
+ * @param {[String]} keys The list of keys.
+ *
+ * @promise
+ *
+ * @header KVAO.keys(filter, callback)
+ */
+module.exports = function keyValueKeys(filter, options, callback) {
+  if (callback === undefined) {
+    if (typeof options === 'function') {
+      callback = options;
+      options = undefined;
+    } else if (options === undefined && typeof filter === 'function') {
+      callback = filter;
+      filter = undefined;
+    }
+  }
+
+  filter = filter || {};
+  options = options || {};
+
+  assert(typeof filter === 'object', 'filter must be an object');
+  assert(typeof options === 'object', 'options must be an object');
+
+  callback = callback || utils.createPromiseCallback();
+
+  var iter = this.iterateKeys(filter, options);
+  var keys = [];
+  iter.next(onNextKey);
+
+  function onNextKey(err, key) {
+    if (err) return callback(err);
+    if (key === undefined) return callback(null, keys);
+    keys.push(key);
+    iter.next(onNextKey);
+  }
+
+  return callback.promise;
+};
+

package.json

@@ -32,6 +32,7 @@
     "node >= 0.6"
   ],
   "devDependencies": {
+    "async-iterators": "^0.2.2",
     "eslint": "^2.5.3",
     "eslint-config-loopback": "^2.0.0",
     "mocha": "^2.1.0",
@@ -44,6 +45,7 @@
     "depd": "^1.0.0",
     "inflection": "^1.6.0",
     "loopback-connector": "^2.1.0",
+    "minimatch": "^3.0.3",
     "node-uuid": "^1.4.2",
     "qs": "^3.1.0",
     "strong-globalize": "^2.6.2",

test/kvao/_helpers.js

@@ -1,9 +1,23 @@
 'use strict';
 
+var Promise = require('bluebird');
+
 exports.givenCacheItem = function(dataSourceFactory) {
   var dataSource = dataSourceFactory();
   return dataSource.createModel('CacheItem', {
     key: String,
     value: 'any',
   });
 };
+
+exports.givenKeys = function(Model, keys, cb) {
+  var p = Promise.all(
+    keys.map(function(k) {
+      return Model.set(k, 'value-' + k);
+    })
+  );
+  if (cb) {
+    p = p.then(function(r) { cb(null, r); }, cb);
+  }
+  return p;
+};

test/kvao/iterate-keys.suite.js

@@ -0,0 +1,48 @@
+'use strict';
+
+var asyncIterators = require('async-iterators');
+var helpers = require('./_helpers');
+var Promise = require('bluebird');
+var should = require('should');
+var toArray = Promise.promisify(asyncIterators.toArray);
+
+module.exports = function(dataSourceFactory, connectorCapabilities) {
+  describe('iterateKeys', function() {
+    var CacheItem;
+    beforeEach(function unpackContext() {
+      CacheItem = helpers.givenCacheItem(dataSourceFactory);
+    });
+
+    it('returns AsyncIterator covering all keys', function() {
+      return helpers.givenKeys(CacheItem, ['key1', 'key2'])
+        .then(function() {
+          var it = CacheItem.iterateKeys();
+          should(it).have.property('next');
+          return toArray(it);
+        })
+        .then(function(keys) {
+          keys.sort();
+          should(keys).eql(['key1', 'key2']);
+        });
+    });
+
+    it('returns AsyncIterator supporting Promises', function() {
+      var iterator;
+      return helpers.givenKeys(CacheItem, ['key'])
+        .then(function() {
+          iterator = CacheItem.iterateKeys();
+          return iterator.next();
+        })
+        .then(function(key) {
+          should(key).equal('key');
+          return iterator.next();
+        })
+        .then(function(key) {
+          // Note: AsyncIterator contract requires `undefined` to signal
+          // the end of the sequence. Other false-y values like `null`
+          // don't work.
+          should(key).equal(undefined);
+        });
+    });
+  });
+};

test/kvao/keys.suite.js

@@ -0,0 +1,105 @@
+'use strict';
+
+var helpers = require('./_helpers');
+var Promise = require('bluebird');
+var should = require('should');
+
+module.exports = function(dataSourceFactory, connectorCapabilities) {
+  describe('keys', function() {
+    var CacheItem;
+    beforeEach(function unpackContext() {
+      CacheItem = helpers.givenCacheItem(dataSourceFactory);
+      CacheItem.sortedKeys = function(filter, options) {
+        return this.keys(filter, options).then(function(keys) {
+          keys.sort();
+          return keys;
+        });
+      };
+    });
+
+    it('returns all keys - Callback API', function(done) {
+      helpers.givenKeys(CacheItem, ['key1', 'key2'], function(err) {
+        if (err) return done(err);
+        CacheItem.keys(function(err, keys) {
+          if (err) return done(err);
+          keys.sort();
+          should(keys).eql(['key1', 'key2']);
+          done();
+        });
+      });
+    });
+
+    it('returns all keys - Promise API', function() {
+      return helpers.givenKeys(CacheItem, ['key1', 'key2'])
+        .then(function() {
+          return CacheItem.keys();
+        })
+        .then(function(keys) {
+          keys.sort();
+          should(keys).eql(['key1', 'key2']);
+        });
+    });
+
+    it('returns keys of the given model only', function() {
+      var AnotherModel = CacheItem.dataSource.createModel('AnotherModel');
+      return helpers.givenKeys(CacheItem, ['key1', 'key2'])
+        .then(function() {
+          return helpers.givenKeys(AnotherModel, ['otherKey1', 'otherKey2']);
+        })
+        .then(function() {
+          return CacheItem.sortedKeys();
+        })
+        .then(function(keys) {
+          should(keys).eql(['key1', 'key2']);
+        });
+    });
+
+    it('handles large key set', function() {
+      var expectedKeys = [];
+      for (var ix = 0; ix < 1000; ix++)
+        expectedKeys.push('key-' + ix);
+      expectedKeys.sort();
+
+      return helpers.givenKeys(CacheItem, expectedKeys)
+        .then(function() {
+          return CacheItem.sortedKeys();
+        })
+        .then(function(keys) {
+          should(keys).eql(expectedKeys);
+        });
+    });
+
+    context('with "filter.match"', function() {
+      beforeEach(function createTestData() {
+        return helpers.givenKeys(CacheItem, [
+          'hallo',
+          'hello',
+          'hxllo',
+          'hllo',
+          'heeello',
+          'foo',
+          'bar',
+        ]);
+      });
+
+      it('supports "?" operator', function() {
+        return CacheItem.sortedKeys({ match: 'h?llo' }).then(function(keys) {
+          should(keys).eql(['hallo', 'hello', 'hxllo']);
+        });
+      });
+
+      it('supports "*" operator', function() {
+        return CacheItem.sortedKeys({ match: 'h*llo' }).then(function(keys) {
+          should(keys).eql(['hallo', 'heeello', 'hello', 'hllo', 'hxllo']);
+        });
+      });
+
+      it('handles no matches found', function() {
+        return CacheItem.sortedKeys({ match: 'not-found' })
+          .then(function(keys) {
+            should(keys).eql([]);
+          });
+      });
+    });
+  });
+};