Merge pull request #24 from bvaughn/performance-tuning-2

Performance tuning

Author: bvaughn

benchmarks/regression-test.js

@@ -24,30 +24,67 @@ var filter = process.argv.length === 3
 var benchmarks = [];
 
 function setupBenchmarks(corpus) {
+  // Index strategies
   initBenchmark({
     corpus,
-    indexStrategy: 'PrefixIndexStrategy',
-    searchIndex: 'UnorderedSearchIndex'
+    indexStrategy: 'PrefixIndexStrategy'
   });
   initBenchmark({
     corpus,
-    indexStrategy: 'AllSubstringsIndexStrategy',
-    searchIndex: 'UnorderedSearchIndex'
+    indexStrategy: 'AllSubstringsIndexStrategy'
   });
   initBenchmark({
     corpus,
-    indexStrategy: 'ExactWordIndexStrategy',
+    indexStrategy: 'PrefixIndexStrategy'
+  });
+
+  // Search indices
+  initBenchmark({
+    corpus,
     searchIndex: 'TfIdfSearchIndex'
   });
   initBenchmark({
     corpus,
-    indexStrategy: 'ExactWordIndexStrategy',
     searchIndex: 'UnorderedSearchIndex'
   });
 
+  // Tokenizers
+  initBenchmark({
+    corpus,
+    tokenizer: 'SimpleTokenizer'
+  });
+  initBenchmark({
+    corpus,
+    tokenizer: 'StemmingTokenizer'
+  });
+  initBenchmark({
+    corpus,
+    tokenizer: 'StopWordsTokenizer'
+  });
+
   runNextTest();
 }
 
+function identity(text) {
+  return text;
+}
+
+function createTokenizer(module, tokenizer) {
+  switch (tokenizer) {
+    case 'SimpleTokenizer':
+      return new module.SimpleTokenizer();
+    case 'StemmingTokenizer':
+      return new module.StemmingTokenizer(
+        identity,
+        new module.SimpleTokenizer()
+      );
+    case 'StopWordsTokenizer':
+      return new module.StopWordsTokenizer(
+        new module.SimpleTokenizer()
+      );
+  }
+}
+
 function createBenchmark() {
   return new Benchmark.Suite()
     .on('cycle', (event) => {
@@ -68,47 +105,48 @@ function runNextTest() {
 
 function initBenchmark({
   corpus,
-  indexStrategy,
-  searchIndex
+  indexStrategy = 'PrefixIndexStrategy',
+  searchIndex = 'UnorderedSearchIndex',
+  tokenizer = 'SimpleTokenizer'
 }) {
-  if (
-    filter &&
-    !indexStrategy.match(filter) &&
-    !searchIndex.match(filter)
-  ) {
-    return;
-  }
-
-  console.log(`Initializing benchmark\t${indexStrategy}\t${searchIndex}`);
-
   initBenchmarkForCreateIndex({
     corpus,
     indexStrategy,
-    searchIndex
+    searchIndex,
+    tokenizer
   });
   initBenchmarkForSearch({
     corpus,
     indexStrategy,
-    searchIndex
+    searchIndex,
+    tokenizer
   });
 }
 
 function initBenchmarkForCreateIndex({
   corpus,
   indexStrategy,
-  searchIndex
+  searchIndex,
+  tokenizer
 }) {
+  var label = `index\t${indexStrategy}\t${searchIndex}\t${tokenizer}`;
+
+  if (filter && !label.match(filter)) {
+    return;
+  }
+
   var benchmark = createBenchmark();
 
   versions.forEach(version => {
     var IndexStrategy = version.module[indexStrategy];
     var Search = version.module.Search;
     var SearchIndex = version.module[searchIndex];
 
-    benchmark.add(`[${version.label}]\tCreate index\t${searchIndex}\t${indexStrategy}`, () => {
+    benchmark.add(`[${version.label}]\t${label}`, () => {
       var search = new Search('isbn');
       search.indexStrategy = new IndexStrategy();
       search.searchIndex = new SearchIndex('isbn');
+      search.tokenizer = createTokenizer(version.module, tokenizer);
       search.addIndex('title');
       search.addIndex('author');
       search.addDocuments(corpus);
@@ -121,8 +159,15 @@ function initBenchmarkForCreateIndex({
 function initBenchmarkForSearch({
   corpus,
   indexStrategy,
-  searchIndex
+  searchIndex,
+  tokenizer
 }) {
+  var label = `search\t${indexStrategy}\t${searchIndex}\t${tokenizer}`;
+
+  if (filter && !label.match(filter)) {
+    return;
+  }
+
   var searchTerms = ['letter', 'world', 'wife', 'love', 'foobar'];
   var searchTermsLength = searchTerms.length;
 
@@ -136,11 +181,12 @@ function initBenchmarkForSearch({
     var search = new Search('isbn');
     search.indexStrategy = new IndexStrategy();
     search.searchIndex = new SearchIndex('isbn');
+    search.tokenizer = createTokenizer(version.module, tokenizer);
     search.addIndex('title');
     search.addIndex('author');
     search.addDocuments(corpus);
 
-    benchmark.add(`[${version.label}]\tSearch strings\t${searchIndex}\t${indexStrategy}`, () => {
+    benchmark.add(`[${version.label}]\t${label}`, () => {
       for (var i = 0, length = searchTermsLength; i < length; i++) {
         search.search(searchTerms[i]);
       }

benchmarks/yarn.lock

@@ -13,6 +13,10 @@ benchmark@^2.1.3:
     lodash "^4.17.3"
     platform "^1.3.3"
 
+[email protected]:
+  version "1.2.1"
+  resolved "https://registry.yarnpkg.com/bindings/-/bindings-1.2.1.tgz#14ad6113812d2d37d72e67b4cacb4bb726505f11"
+
 js-search@^1.3.5:
   version "1.3.5"
   resolved "https://registry.yarnpkg.com/js-search/-/js-search-1.3.5.tgz#b489c05aabc4b22cdb75484fe75ac49fa94feff2"
@@ -25,6 +29,17 @@ lunr@^0.7.2:
   version "0.7.2"
   resolved "https://registry.yarnpkg.com/lunr/-/lunr-0.7.2.tgz#79a30e932e216cba163541ee37a3607c12cd7281"
 
+microtime@^2.1.2:
+  version "2.1.2"
+  resolved "https://registry.yarnpkg.com/microtime/-/microtime-2.1.2.tgz#9c955d0781961ab13a1b6f9a82b080f5d7ecd83b"
+  dependencies:
+    bindings "1.2.x"
+    nan "2.4.x"
+
+[email protected]:
+  version "2.4.0"
+  resolved "https://registry.yarnpkg.com/nan/-/nan-2.4.0.tgz#fb3c59d45fe4effe215f0b890f8adf6eb32d2232"
+
 platform@^1.3.3:
   version "1.3.3"
   resolved "https://registry.yarnpkg.com/platform/-/platform-1.3.3.tgz#646c77011899870b6a0903e75e997e8e51da7461"

source/IndexStrategy/AllSubstringsIndexStrategy.js

@@ -12,10 +12,14 @@ export class AllSubstringsIndexStrategy implements IIndexStrategy {
    */
   expandToken(token : string) : Array<string> {
     var expandedTokens = [];
+    var string;
 
     for (var i = 0, length = token.length; i < length; ++i) {
+      string = '';
+
       for (var j = i; j < length; ++j) {
-        expandedTokens.push(token.substring(i, j + 1));
+        string += token.charAt(j);
+        expandedTokens.push(string);
       }
     }
 

source/IndexStrategy/PrefixIndexStrategy.js

@@ -12,9 +12,11 @@ export class PrefixIndexStrategy implements IIndexStrategy {
    */
   expandToken(token : string) : Array<string> {
     var expandedTokens = [];
+    var string = '';
 
     for (var i = 0, length = token.length; i < length; ++i) {
-      expandedTokens.push(token.substring(0, i + 1));
+      string += token.charAt(i);
+      expandedTokens.push(string);
     }
 
     return expandedTokens;

source/Search.js

@@ -25,7 +25,7 @@ export class Search {
   /**
    * Array containing either a property name or a path (list of property names) to a nested value
    */
-  searchableFields : Array<string|Array<string>>;
+  _searchableFields : Array<string|Array<string>>;
 
   _searchIndex : ISearchIndex;
   _tokenizer : ITokenizer;
@@ -46,7 +46,7 @@ export class Search {
     this._tokenizer = new SimpleTokenizer();
 
     this._documents = [];
-    this.searchableFields = [];
+    this._searchableFields = [];
   }
 
   /**
@@ -128,7 +128,7 @@ export class Search {
    */
   addDocuments(documents : Array<Object>) : void {
     this._documents = this._documents.concat(documents);
-    this.indexDocuments_(documents, this.searchableFields);
+    this.indexDocuments_(documents, this._searchableFields);
   }
 
   /**
@@ -137,7 +137,7 @@ export class Search {
    * @param field Searchable field or field path. Pass a string to index a top-level field and an array of strings for nested fields.
    */
   addIndex(field : string|Array<string>) {
-    this.searchableFields.push(field);
+    this._searchableFields.push(field);
     this.indexDocuments_(this._documents, [field]);
   }
 
@@ -154,24 +154,30 @@ export class Search {
 
   /**
    * @param documents
-   * @param searchableFields Array containing property names and paths (lists of property names) to nested values
+   * @param _searchableFields Array containing property names and paths (lists of property names) to nested values
    * @private
    */
-  indexDocuments_(documents : Array<Object>, searchableFields : Array<string|Array<string>>) : void {
+  indexDocuments_(documents : Array<Object>, _searchableFields : Array<string|Array<string>>) : void {
     this._initialized = true;
 
+    var indexStrategy = this._indexStrategy;
+    var sanitizer = this._sanitizer;
+    var searchIndex = this._searchIndex;
+    var tokenizer = this._tokenizer;
+    var uidFieldName = this._uidFieldName;
+
     for (var di = 0, numDocuments = documents.length; di < numDocuments; di++) {
-      var document : Object = documents[di];
-      var uid : string = document[this._uidFieldName];
+      var doc = documents[di];
+      var uid = doc[uidFieldName];
 
-      for (var sfi = 0, numSearchableFields = searchableFields.length; sfi < numSearchableFields; sfi++) {
-        var fieldValue : any;
-        var searchableField : string|Array<string> = searchableFields[sfi];
+      for (var sfi = 0, numSearchableFields = _searchableFields.length; sfi < numSearchableFields; sfi++) {
+        var fieldValue;
+        var searchableField = _searchableFields[sfi];
 
         if (searchableField instanceof Array) {
-          fieldValue = Search.getNestedFieldValue(document, searchableField);
+          fieldValue = getNestedFieldValue(doc, searchableField);
         } else {
-          fieldValue = document[searchableField];
+          fieldValue = doc[searchableField];
         }
 
         if (
@@ -183,45 +189,45 @@ export class Search {
         }
 
         if (typeof fieldValue === 'string') {
-          var fieldTokens : Array<string> = this._tokenizer.tokenize(this._sanitizer.sanitize(fieldValue));
+          var fieldTokens = tokenizer.tokenize(sanitizer.sanitize(fieldValue));
 
           for (var fti = 0, numFieldValues = fieldTokens.length; fti < numFieldValues; fti++) {
-            var fieldToken : string = fieldTokens[fti];
-            var expandedTokens : Array<string> = this._indexStrategy.expandToken(fieldToken);
+            var fieldToken = fieldTokens[fti];
+            var expandedTokens = indexStrategy.expandToken(fieldToken);
 
             for (var eti = 0, nummExpandedTokens = expandedTokens.length; eti < nummExpandedTokens; eti++) {
               var expandedToken = expandedTokens[eti];
 
-              this._searchIndex.indexDocument(expandedToken, uid, document);
+              searchIndex.indexDocument(expandedToken, uid, doc);
             }
           }
         }
       }
     }
   }
+}
 
-  /**
-   * Find and return a nested object value.
-   *
-   * @param object to crawl
-   * @param path Property path
-   * @returns {any}
-   */
-  static getNestedFieldValue(object : Object, path : Array<string>) {
-    path = path || [];
-    object = object || {};
+/**
+ * Find and return a nested object value.
+ *
+ * @param object to crawl
+ * @param path Property path
+ * @returns {any}
+ */
+function getNestedFieldValue(object : Object, path : Array<string>) {
+  path = path || [];
+  object = object || {};
 
-    var value = object;
+  var value = object;
 
-    // walk down the property path
-    for (var i = 0; i < path.length; i++) {
-      value = value[path[i]];
+  // walk down the property path
+  for (var i = 0; i < path.length; i++) {
+    value = value[path[i]];
 
-      if (value == null) {
-        return null;
-      }
+    if (value == null) {
+      return null;
     }
-
-    return value;
   }
+
+  return value;
 }