feat(*): enable addons support for enhanced language model

closes #7
references #6

Author: sanjayaksaxena

src/allowed.js

@@ -48,7 +48,8 @@ allowed.its4token = new Set( [
   its.abbrevFlag,
   its.suffix,
   its.type,
-  its.value
+  its.value,
+  its.stem
 ] );
 
 allowed.its4tokens = allowed.its4token;
@@ -91,7 +92,8 @@ allowed.its4sentence = new Set( [
   its.span,
   its.markedUpText,
   its.negationFlag,
-  its.sentiment
+  its.sentiment,
+  its.stem
 ] );
 
 allowed.its4document = allowed.its4sentence;

src/api/col-sentences-out.js

@@ -38,14 +38,14 @@ var itmSentenceOut = require( './itm-sentence-out.js' );
  * data type or data structure.
  * @param  {object}   rdd          Raw Document Data-structure.
  * @param  {function} itsf         Desired `its` mapper.
- * @param  {object}   wordVectors  The word vectors.
+ * @param  {object}   addons       The model's addons.
  * @return {*}                     Mapped sentences.
  * @private
  */
-var colSentencesOut = function ( rdd, itsf, wordVectors ) {
+var colSentencesOut = function ( rdd, itsf, addons ) {
   var sents = [];
   for ( let i = 0; i < rdd.sentences.length; i += 1 ) {
-    sents.push( itmSentenceOut( i, rdd, itsf, wordVectors ) );
+    sents.push( itmSentenceOut( i, rdd, itsf, addons ) );
   }
   return sents;
 }; // colSentencesOut()

src/api/col-tokens-out.js

@@ -48,14 +48,14 @@ var psMask = constants.psMask;
  * @param  {object}   rdd         Raw Document Data-structure.
  * @param  {function} itsf        Desired `its` mapper.
  * @param  {function} asf         Desired `as` reducer.
- * @param  {object}   wordVectors The word vectors.
+ * @param  {object}   addons      The model's addons.
  * @return {*}                    Map-reduced collection of tokens.
  * @private
  */
-var colTokensOut = function ( start, end, rdd, itsf, asf, wordVectors ) {
+var colTokensOut = function ( start, end, rdd, itsf, asf, addons ) {
   // Vectors require completely different handling.
   if ( itsf === its.vector ) {
-    return its.vector( start, end, rdd.tokens, wordVectors );
+    return its.vector( start, end, rdd.tokens, addons );
   }
 
   // Not a vector request, perform map-reduce.
@@ -65,11 +65,11 @@ var colTokensOut = function ( start, end, rdd, itsf, asf, wordVectors ) {
   // Note, `as.text/markedUpText` needs special attention to include preceeding spaces.
   if ( asfn === as.text || asfn === as.markedUpText ) {
     for ( let i = start; i <= end; i += 1 ) {
-      mappedTkns.push( ''.padEnd( rdd.tokens[ ( i * tkSize ) + 1 ] & psMask ), itsf( i, rdd.tokens, rdd.cache ) );  // eslint-disable-line no-bitwise
+      mappedTkns.push( ''.padEnd( rdd.tokens[ ( i * tkSize ) + 1 ] & psMask ), itsf( i, rdd.tokens, rdd.cache, addons ) );  // eslint-disable-line no-bitwise
     }
   } else {
     for ( let i = start; i <= end; i += 1 ) {
-      mappedTkns.push( itsfn( i, rdd.tokens, rdd.cache ) );
+      mappedTkns.push( itsfn( i, rdd.tokens, rdd.cache, addons ) );
     }
   }
 

src/api/itm-document-out.js

@@ -41,15 +41,15 @@ var colTokensOut = require( './col-tokens-out.js' );
  * data type or data structure.
  * @param  {Object}   rdd         Raw Document Data-structure.
  * @param  {function} itsf        Desired `its` mapper.
- * @param  {Object}   wordVectors The word vectors.
+ * @param  {Object}   addons      The model's addons.
  * @return {*}                    Mapped value.
  * @private
  */
-var itmDocumentOut = function ( rdd, itsf, wordVectors ) {
+var itmDocumentOut = function ( rdd, itsf, addons ) {
   var document = rdd.document;
   // Vectors require completely different handling.
   if ( itsf === its.vector ) {
-    return its.vector( document, rdd, wordVectors );
+    return its.vector( document, rdd, addons );
   }
 
   var itsfn = ( itsf && allowed.its4document.has( itsf ) ) ? itsf : its.value;
@@ -67,7 +67,7 @@ var itmDocumentOut = function ( rdd, itsf, wordVectors ) {
   // returned the `value`. Refer to `its.markedUpText`.
   var asfn = ( itsfn === its.markedUpText ) ? as.markedUpText : as.text;
 
-  return colTokensOut( document[ 0 ], document[ 1 ], rdd, itsfn, asfn );
+  return colTokensOut( document[ 0 ], document[ 1 ], rdd, itsfn, asfn, addons );
 }; // itmDocumentOut()
 
 module.exports = itmDocumentOut;

src/api/itm-sentence-out.js

@@ -42,15 +42,15 @@ var colTokensOut = require( './col-tokens-out.js' );
  * @param  {number}   index       The index of desired sentence.
  * @param  {Object}   rdd         Raw Document Data-structure.
  * @param  {function} itsf        Desired `its` mapper.
- * @param  {object}   wordVectors The word vectors.
+ * @param  {object}   addons      The model's addons.
  * @return {*}                    Mapped value.
  * @private
  */
-var itmSentenceOut = function ( index, rdd, itsf, wordVectors ) {
+var itmSentenceOut = function ( index, rdd, itsf, addons ) {
   var sentence = rdd.sentences[ index ];
   // Vectors require completely different handling.
   if ( itsf === its.vector ) {
-    return its.vector( sentence, rdd, wordVectors );
+    return its.vector( sentence, rdd, addons );
   }
 
   var itsfn = ( itsf && allowed.its4sentence.has( itsf ) ) ? itsf : its.value;
@@ -68,7 +68,7 @@ var itmSentenceOut = function ( index, rdd, itsf, wordVectors ) {
   // returned the `value`. Refer to `its.markedUpText`.
   var asfn = ( itsfn === its.markedUpText ) ? as.markedUpText : as.text;
 
-  return colTokensOut( sentence[ 0 ], sentence[ 1 ], rdd, itsfn, asfn );
+  return colTokensOut( sentence[ 0 ], sentence[ 1 ], rdd, itsfn, asfn, addons );
 }; // itmSentenceOut()
 
 module.exports = itmSentenceOut;

src/api/itm-token-out.js

@@ -40,18 +40,18 @@ var allowed = require( '../allowed.js' );
  * @param  {number}   index       The index of desired token.
  * @param  {Object}   rdd         Raw Document Data-structure.
  * @param  {function} itsf        Desired `its` mapper.
- * @param  {object}   wordVectors The word vectors.
+ * @param  {object}   addons      The model's addons.
  * @return {*}                    Mapped value.
  * @private
  */
-var itmTokenOut = function ( index, rdd, itsf, wordVectors ) {
+var itmTokenOut = function ( index, rdd, itsf, addons ) {
   // Vectors require completely different handling.
   if ( itsf === its.vector ) {
-    return its.vector( index, rdd, wordVectors );
+    return its.vector( index, rdd, addons );
   }
   // Not a vector request, map using `itsf`.
   var f = ( allowed.its4token.has( itsf ) ) ? itsf : its.value;
-  return f( index, rdd.tokens, rdd.cache );
+  return f( index, rdd.tokens, rdd.cache, addons );
 }; // itmTokenOut()
 
 module.exports = itmTokenOut;

src/api/sel-tokens-out.js

@@ -47,14 +47,14 @@ var psMask = constants.psMask;
  * @param  {obejct}   rdd         Raw document data structure.
  * @param  {function} itsf        Desired `its` mapper.
  * @param  {function} asf         Desired `as` reducer.
- * @param  {object}   wordVectors The word vectors.
+ * @param  {object}   addons      The addons from the model.
  * @return {*}                    Reduced value.
  * @private
  */
-var selTokensOut = function ( selTokens, rdd, itsf, asf, wordVectors ) {
+var selTokensOut = function ( selTokens, rdd, itsf, asf, addons ) {
   // Vectors require completely different handling.
   if ( itsf === its.vector ) {
-    return its.vector( selTokens, rdd.tokens, wordVectors );
+    return its.vector( selTokens, rdd.tokens, addons );
   }
 
   // Not a vector request, perform map-reduce.
@@ -66,11 +66,11 @@ var selTokensOut = function ( selTokens, rdd, itsf, asf, wordVectors ) {
   // No `markedUpText` allowed here.
   if ( asfn === as.text ) {
     for ( let i = 0; i < selTokens.length; i += 1 ) {
-      mappedTkns.push( ''.padEnd( rdd.tokens[ ( selTokens[ i ] * tkSize ) + 1 ] & psMask ), itsf( selTokens[ i ], rdd.tokens, rdd.cache ) );  // eslint-disable-line no-bitwise
+      mappedTkns.push( ''.padEnd( rdd.tokens[ ( selTokens[ i ] * tkSize ) + 1 ] & psMask ), itsf( selTokens[ i ], rdd.tokens, rdd.cache, addons ) );  // eslint-disable-line no-bitwise
     }
   } else {
     for ( let i = 0; i < selTokens.length; i += 1 ) {
-      mappedTkns.push( itsfn( selTokens[ i ], rdd.tokens, rdd.cache ) );
+      mappedTkns.push( itsfn( selTokens[ i ], rdd.tokens, rdd.cache, addons ) );
     }
   }
 

src/doc-v2.js

@@ -79,11 +79,11 @@ var printTokens = require( './api/print-tokens.js' );
  * developer APIs.
  *
  * @param  {object} docData     It encapsulates the document data.
- * @param  {object} wordVectors Word vector, optional.
+ * @param  {object} addons      The model's addon, may contain word vectors, stemmer etc.
  * @return {object}             conatining APIs.
  * @private
  */
-var doc = function ( docData, wordVectors ) {
+var doc = function ( docData, addons ) {
   // Extract `cache` as it is frequently accessed.
   var cache = docData.cache;
 
@@ -158,7 +158,7 @@ var doc = function ( docData, wordVectors ) {
     // Markup this token.
     api.markup = ( beginMarker, endMarker ) => markings.push( [ index, index, beginMarker, endMarker ] );
     // Output this token or its properties using mapper function — `f`.
-    api.out = ( f ) => itmTokenOut( index, docData, f, wordVectors );
+    api.out = ( f ) => itmTokenOut( index, docData, f, addons );
     // Access the parent sentence.
     api.parentSentence = () => getParentItem( index, sentences, itemSentence );
     // Index within the document.
@@ -188,7 +188,7 @@ var doc = function ( docData, wordVectors ) {
     api.length = () => ( selectedTokens.length );
     // Output this collection of selected tokens as a reduced values or properties
     // using map/reduce functions — `f/g`.
-    api.out = ( f, g ) => selTokensOut( selectedTokens, docData, f, g, wordVectors );
+    api.out = ( f, g ) => selTokensOut( selectedTokens, docData, f, g, addons );
     return api;
   }; // colTokens()
 
@@ -217,7 +217,7 @@ var doc = function ( docData, wordVectors ) {
         api.length = () => ( end - start + 1 );
         // Output this token collection as a reduced values or properties using
         // map/reduce functions — `f/g`.
-        api.out = ( f, g ) => colTokensOut( start, end, docData, f, g, wordVectors );
+        api.out = ( f, g ) => colTokensOut( start, end, docData, f, g, addons );
 
         return api;
       }
@@ -409,7 +409,7 @@ var doc = function ( docData, wordVectors ) {
     // Markup this sentence.
     api.markup = ( beginMarker, endMarker ) => markings.push( [ sentences[ index ][ 0 ], sentences[ index ][ 1 ], beginMarker, endMarker ] );
     // Output this sentence as text.
-    api.out = ( f ) => itmSentenceOut( index, docData, f, wordVectors );
+    api.out = ( f ) => itmSentenceOut( index, docData, f, addons );
     // Outputs the collection of entities, if any, contained in this sentence.
     api.entities = () => colSelectedEntities( containedEntities( entities, sentences[ index ][ 0 ], sentences[ index ][ 1 ] ) );
     // Outputs the collection of custom entities, if any, contained in this sentence.
@@ -438,7 +438,7 @@ var doc = function ( docData, wordVectors ) {
     // Length of this collection.
     api.length = () => ( sentences.length );
     // Output this collection of sentences as an array of strings.
-    api.out = ( f ) => colSentencesOut( docData, f, wordVectors );
+    api.out = ( f ) => colSentencesOut( docData, f, addons );
     return api;
   }; // colSentences()
 
@@ -450,7 +450,7 @@ var doc = function ( docData, wordVectors ) {
   methods.customEntities = colCustomEntities;
   methods.isLexeme = isLexeme;
   methods.isOOV = cache.isOOV;
-  methods.out = ( f ) => itmDocumentOut( docData, f, wordVectors );
+  methods.out = ( f ) => itmDocumentOut( docData, f, addons );
   methods.sentences = colSentences;
   methods.tokens = colTokens( 0, docData.numOfTokens - 1 );
 

src/its.js

@@ -111,6 +111,10 @@ its.value = function ( index, tokens, cache ) {
   return cache.value( tokens[ index * tkSize ] );
 }; // value()
 
+its.stem = function ( index, tokens, cache, addons ) {
+  return addons.stem( cache.value( tokens[ index * tkSize ] ) );
+}; // value()
+
 its.vector = function ( ) {
   return ( new Array( 100 ).fill( 0 ) );
 }; // vector()

src/wink-nlp.js

@@ -117,7 +117,7 @@ var nlp = function ( theModel ) {
    * @returns {void} nothing!.
    * @private
   */
-  var load = function (  ) {
+  var load = function () {
     // Load language model.
     model = theModel.core();
     // With `intrinsicSize` captured, instantiate cache etc.
@@ -292,7 +292,7 @@ var nlp = function ( theModel ) {
     // }
 
     // Now create the document!
-    var doc = Doc( rdd, theModel.wordVectors ); // eslint-disable-line new-cap
+    var doc = Doc( rdd, theModel.addons ); // eslint-disable-line new-cap
 
     // All done — cleanup document's data.
     wrappedDocData.clean();
@@ -361,7 +361,7 @@ var nlp = function ( theModel ) {
   }
 
   // Load the model.
-  load( theModel );
+  load();
   // Setup default configuration.
   // definePipeConfig();
   // Methods.