250 precursor (#293)

* more accurate paramToJson in parameter_hunter

* updating unit tests for parameter_hunter paramToJson

* minor edit

* another param string parser

* bugs fixed, unit tests working

* variable name change

* Added (failing for now) test to highlight the bug.

* Added another test for another bug (parenthesis in value confuses patternhunter)

* Added additional parameter hunter tests.

* replacing JSON with JSON5 for better error messaging

* working paramToJson

* stricter unit tests

* comment updates

* simplfying paramToJson further

* simplfying paramToJson further

* comment update

* tightened parameter key search

* comment update

* accommodating decimals and exponentials as valid numeric values

* comment update

* fixing array instead of number typo

* more comments

* spacing to pass eslint

* removing unnecessary escape

* fixing substring param typo

* renaming json5 instances

* comment update

Author: e2tha-e

.gitignore

@@ -1,4 +1,5 @@
 node_modules/
+npm-debug.log
 .DS_Store
 latest-change.txt
 patternlab.json

core/lib/list_item_hunter.js

@@ -13,6 +13,7 @@
 var list_item_hunter = function () {
 
   var extend = require('util')._extend,
+    JSON5 = require('json5'),
     pa = require('./pattern_assembler'),
     smh = require('./style_modifier_hunter'),
     pattern_assembler = new pa(),
@@ -44,7 +45,13 @@ var list_item_hunter = function () {
         }
 
         //check for a local listitems.json file
-        var listData = JSON.parse(JSON.stringify(patternlab.listitems));
+        var listData;
+        try {
+          listData = JSON5.parse(JSON5.stringify(patternlab.listitems));
+        } catch (err) {
+          console.log('There was an error parsing JSON for ' + pattern.abspath);
+          console.log(err);
+        }
         listData = pattern_assembler.merge_data(listData, pattern.listitems);
 
         //iterate over each copied block, rendering its contents along with pattenlab.listitems[i]
@@ -54,8 +61,15 @@ var list_item_hunter = function () {
 
           //combine listItem data with pattern data with global data
           var itemData = listData['' + items.indexOf(loopNumberString)]; //this is a property like "2"
-          var globalData = JSON.parse(JSON.stringify(patternlab.data));
-          var localData = JSON.parse(JSON.stringify(pattern.jsonFileData));
+          var globalData;
+          var localData;
+          try {
+            globalData = JSON5.parse(JSON5.stringify(patternlab.data));
+            localData = JSON5.parse(JSON5.stringify(pattern.jsonFileData));
+          } catch (err) {
+            console.log('There was an error parsing JSON for ' + pattern.abspath);
+            console.log(err);
+          }
 
           var allData = pattern_assembler.merge_data(globalData, localData);
           allData = pattern_assembler.merge_data(allData, itemData !== undefined ? itemData[i] : {}); //itemData could be undefined if the listblock contains no partial, just markup
@@ -71,7 +85,13 @@ var list_item_hunter = function () {
               var partialPattern = pattern_assembler.get_pattern_by_key(partialName, patternlab);
 
               //create a copy of the partial so as to not pollute it after the get_pattern_by_key call.
-              var cleanPartialPattern = JSON.parse(JSON.stringify(partialPattern));
+              var cleanPartialPattern;
+              try {
+                cleanPartialPattern = JSON5.parse(JSON5.stringify(partialPattern));
+              } catch (err) {
+                console.log('There was an error parsing JSON for ' + pattern.abspath);
+                console.log(err);
+              }
 
               //if partial has style modifier data, replace the styleModifier value
               if (foundPartials[j].indexOf(':') > -1) {

core/lib/parameter_hunter.js

@@ -13,110 +13,230 @@
 var parameter_hunter = function () {
 
   var extend = require('util')._extend,
+    JSON5 = require('json5'),
     pa = require('./pattern_assembler'),
     smh = require('./style_modifier_hunter'),
-    style_modifier_hunter = new smh(),
-    pattern_assembler = new pa();
-
+    pattern_assembler = new pa(),
+    style_modifier_hunter = new smh();
+
+  /**
+   * This function is really to accommodate the lax JSON-like syntax allowed by
+   * Pattern Lab PHP for parameter submissions to partials. Unfortunately, no
+   * easily searchable library was discovered for this. What we had to do was
+   * write a custom script to crawl through the parameter string, and wrap the
+   * keys and values in double-quotes as necessary.
+   * The steps on a high-level are as follows:
+   *   * Further escape all escaped quotes and colons. Use the string
+   *     representation of their unicodes for this. This has the added bonus
+   *     of being interpreted correctly by JSON5.parse() without further
+   *     modification. This will be useful later in the function.
+   *   * Once escaped quotes are out of the way, we know the remaining quotes
+   *     are either key/value wrappers or wrapped within those wrappers. We know
+   *     that remaining commas and colons are either delimiters, or wrapped
+   *     within quotes to not be recognized as such.
+   *   * A do-while loop crawls paramString to write keys to a keys array and
+   *     values to a values array.
+   *   * Start by parsing the first key. Determine the type of wrapping quote,
+   *     if any.
+   *   * By knowing the open wrapper, we know that the next quote of that kind
+   *     (if the key is wrapped in quotes), HAS to be the close wrapper.
+   *     Similarly, if the key is unwrapped, we know the next colon HAS to be
+   *     the delimiter between key and value.
+   *   * Save the key to the keys array.
+   *   * Next, search for a value. It will either be the next block wrapped in
+   *     quotes, or a string of alphanumerics, decimal points, or minus signs.
+   *   * Save the value to the values array.
+   *   * The do-while loop truncates the paramString value while parsing. Its
+   *     condition for completion is when the paramString is whittled down to an
+   *     empty string.
+   *   * After the keys and values arrays are built, a for loop iterates through
+   *     them to build the final paramStringWellFormed string.
+   *   * No quote substitution had been done prior to this loop. In this loop,
+   *     all keys are ensured to be wrapped in double-quotes. String values are
+   *     also ensured to be wrapped in double-quotes.
+   *   * Unescape escaped unicodes except for double-quotes. Everything beside
+   *     double-quotes will be wrapped in double-quotes without need for escape.
+   *   * Return paramStringWellFormed.
+   *
+   * @param {string} pString
+   * @returns {string} paramStringWellFormed
+   */
   function paramToJson(pString) {
-    var paramStringWellFormed = '';
-    var paramStringTmp;
-    var colonPos;
-    var delimitPos;
-    var quotePos;
-    var paramString = pString;
-
+    var colonPos = -1;
+    var keys = [];
+    var paramString = pString; // to not reassign param
+    var paramStringWellFormed;
+    var quotePos = -1;
+    var regex;
+    var values = [];
+    var wrapper;
+
+    //replace all escaped double-quotes with escaped unicode
+    paramString = paramString.replace(/\\"/g, '\\u0022');
+
+    //replace all escaped single-quotes with escaped unicode
+    paramString = paramString.replace(/\\'/g, '\\u0027');
+
+    //replace all escaped colons with escaped unicode
+    paramString = paramString.replace(/\\:/g, '\\u0058');
+
+    //with escaped chars out of the way, crawl through paramString looking for
+    //keys and values
     do {
 
-      //if param key is wrapped in single quotes, replace with double quotes.
-      paramString = paramString.replace(/(^\s*[\{|\,]\s*)'([^']+)'(\s*\:)/, '$1"$2"$3');
+      //check if searching for a key
+      if (paramString[0] === '{' || paramString[0] === ',') {
+        paramString = paramString.substring(1, paramString.length).trim();
 
-      //if params key is not wrapped in any quotes, wrap in double quotes.
-      paramString = paramString.replace(/(^\s*[\{|\,]\s*)([^\s"'\:]+)(\s*\:)/, '$1"$2"$3');
+        //search for end quote if wrapped in quotes. else search for colon.
+        //everything up to that position will be saved in the keys array.
+        switch (paramString[0]) {
 
-      //move param key to paramStringWellFormed var.
-      colonPos = paramString.indexOf(':');
+          //need to search for end quote pos in case the quotes wrap a colon
+          case '"':
+          case '\'':
+            wrapper = paramString[0];
+            quotePos = paramString.indexOf(wrapper, 1);
+            break;
 
-      //except to prevent infinite loops.
-      if (colonPos === -1) {
-        colonPos = paramString.length - 1;
-      }
-      else {
-        colonPos += 1;
-      }
-      paramStringWellFormed += paramString.substring(0, colonPos);
-      paramString = paramString.substring(colonPos, paramString.length).trim();
+          default:
+            colonPos = paramString.indexOf(':');
+        }
 
-      //if param value is wrapped in single quotes, replace with double quotes.
-      if (paramString[0] === '\'') {
-        quotePos = paramString.search(/[^\\]'/);
+        if (quotePos > -1) {
+          keys.push(paramString.substring(0, quotePos + 1).trim());
 
-        //except for unclosed quotes to prevent infinite loops.
-        if (quotePos === -1) {
-          quotePos = paramString.length - 1;
-        }
-        else {
-          quotePos += 2;
-        }
+          //truncate the beginning from paramString and look for a value
+          paramString = paramString.substring(quotePos + 1, paramString.length).trim();
 
-        //prepare param value for move to paramStringWellFormed var.
-        paramStringTmp = paramString.substring(0, quotePos);
+          //unset quotePos
+          quotePos = -1;
 
-        //unescape any escaped single quotes.
-        paramStringTmp = paramStringTmp.replace(/\\'/g, '\'');
+        } else if (colonPos > -1) {
+          keys.push(paramString.substring(0, colonPos).trim());
 
-        //escape any double quotes.
-        paramStringTmp = paramStringTmp.replace(/"/g, '\\"');
+          //truncate the beginning from paramString and look for a value
+          paramString = paramString.substring(colonPos, paramString.length);
 
-        //replace the delimiting single quotes with double quotes.
-        paramStringTmp = paramStringTmp.replace(/^'/, '"');
-        paramStringTmp = paramStringTmp.replace(/'$/, '"');
+          //unset colonPos
+          colonPos = -1;
 
-        //move param key to paramStringWellFormed var.
-        paramStringWellFormed += paramStringTmp;
-        paramString = paramString.substring(quotePos, paramString.length).trim();
+        //if there are no more colons, and we're looking for a key, there is
+        //probably a problem. stop any further processing.
+        } else {
+          paramString = '';
+          break;
+        }
       }
 
-      //if param value is wrapped in double quotes, just move to paramStringWellFormed var.
-      else if (paramString[0] === '"') {
-        quotePos = paramString.search(/[^\\]"/);
-
-        //except for unclosed quotes to prevent infinite loops.
-        if (quotePos === -1) {
-          quotePos = paramString.length - 1;
+      //now, search for a value
+      if (paramString[0] === ':') {
+        paramString = paramString.substring(1, paramString.length).trim();
+
+        //the only reason we're using regexes here, instead of indexOf(), is
+        //because we don't know if the next delimiter is going to be a comma or
+        //a closing curly brace. since it's not much of a performance hit to
+        //use regexes as sparingly as here, and it's much more concise and
+        //readable, we'll use a regex for match() and replace() instead of
+        //performing conditional logic with indexOf().
+        switch (paramString[0]) {
+
+          //since a quote of same type as its wrappers would be escaped, and we
+          //escaped those even further with their unicodes, it is safe to look
+          //for wrapper pairs and conclude that their contents are values
+          case '"':
+            regex = /^"(.|\s)*?"/;
+            break;
+          case '\'':
+            regex = /^'(.|\s)*?'/;
+            break;
+
+          //if there is no value wrapper, regex for alphanumerics, decimal
+          //points, and minus signs for exponential notation.
+          default:
+            regex = /^[\w\-\.]*/;
         }
-        else {
-          quotePos += 2;
+        values.push(paramString.match(regex)[0].trim());
+
+        //truncate the beginning from paramString and continue either
+        //looking for a key, or returning 
+        paramString = paramString.replace(regex, '').trim();
+
+        //exit do while if the final char is '}'
+        if (paramString === '}') {
+          paramString = '';
+          break;
         }
 
-        //move param key to paramStringWellFormed var.
-        paramStringWellFormed += paramString.substring(0, quotePos);
-        paramString = paramString.substring(quotePos, paramString.length).trim();
+      //if there are no more colons, and we're looking for a value, there is
+      //probably a problem. stop any further processing.
+      } else {
+        paramString = '';
+        break;
       }
+    } while (paramString);
 
-      //if param value is not wrapped in quotes, move everthing up to the delimiting comma to paramStringWellFormed var.
-      else {
-        delimitPos = paramString.indexOf(',');
-
-        //except to prevent infinite loops.
-        if (delimitPos === -1) {
-          delimitPos = paramString.length - 1;
+    //build paramStringWellFormed string for JSON parsing
+    paramStringWellFormed = '{';
+    for (var i = 0; i < keys.length; i++) {
+
+      //keys
+      //replace single-quote wrappers with double-quotes
+      if (keys[i][0] === '\'' && keys[i][keys[i].length - 1] === '\'') {
+        paramStringWellFormed += '"';
+
+        //any enclosed double-quotes must be escaped
+        paramStringWellFormed += keys[i].substring(1, keys[i].length - 1).replace(/"/g, '\\"');
+        paramStringWellFormed += '"';
+      } else {
+
+        //open wrap with double-quotes if no wrapper
+        if (keys[i][0] !== '"' && keys[i][0] !== '\'') {
+          paramStringWellFormed += '"';
+
+          //this is to clean up vestiges from Pattern Lab PHP's escaping scheme.
+          //F.Y.I. Pattern Lab PHP would allow special characters like question
+          //marks in parameter keys so long as the key was unwrapped and the
+          //special character escaped with a backslash. In Node, we need to wrap
+          //those keys and unescape those characters.
+          keys[i] = keys[i].replace(/\\/g, '');
         }
-        else {
-          delimitPos += 1;
+
+        paramStringWellFormed += keys[i];
+
+        //close wrap with double-quotes if no wrapper
+        if (keys[i][keys[i].length - 1] !== '"' && keys[i][keys[i].length - 1] !== '\'') {
+          paramStringWellFormed += '"';
         }
-        paramStringWellFormed += paramString.substring(0, delimitPos);
-        paramString = paramString.substring(delimitPos, paramString.length).trim();
       }
 
-      //break at the end.
-      if (paramString.length === 1) {
-        paramStringWellFormed += paramString.trim();
-        paramString = '';
-        break;
+      //colon delimiter.
+      paramStringWellFormed += ':'; + values[i];
+
+      //values
+      //replace single-quote wrappers with double-quotes
+      if (values[i][0] === '\'' && values[i][values[i].length - 1] === '\'') {
+        paramStringWellFormed += '"';
+
+        //any enclosed double-quotes must be escaped
+        paramStringWellFormed += values[i].substring(1, values[i].length - 1).replace(/"/g, '\\"');
+        paramStringWellFormed += '"';
+
+      //for everything else, just add the value however it's wrapped
+      } else {
+        paramStringWellFormed += values[i];
       }
 
-    } while (paramString);
+      //comma delimiter
+      if (i < keys.length - 1) {
+        paramStringWellFormed += ',';
+      }
+    }
+    paramStringWellFormed += '}';
+
+    //unescape escaped unicode except for double-quotes
+    paramStringWellFormed = paramStringWellFormed.replace(/\\u0027/g, '\'');
+    paramStringWellFormed = paramStringWellFormed.replace(/\\u0058/g, ':');
 
     return paramStringWellFormed;
   }
@@ -140,7 +260,7 @@ var parameter_hunter = function () {
 
         //strip out the additional data, convert string to JSON.
         var leftParen = pMatch.indexOf('(');
-        var rightParen = pMatch.indexOf(')');
+        var rightParen = pMatch.lastIndexOf(')');
         var paramString = '{' + pMatch.substring(leftParen + 1, rightParen) + '}';
         var paramStringWellFormed = paramToJson(paramString);
 
@@ -149,11 +269,12 @@ var parameter_hunter = function () {
         var localData = {};
 
         try {
-          paramData = JSON.parse(paramStringWellFormed);
-          globalData = JSON.parse(JSON.stringify(patternlab.data));
-          localData = JSON.parse(JSON.stringify(pattern.jsonFileData || {}));
-        } catch (e) {
-          console.log(e);
+          paramData = JSON5.parse(paramStringWellFormed);
+          globalData = JSON5.parse(JSON5.stringify(patternlab.data));
+          localData = JSON5.parse(JSON5.stringify(pattern.jsonFileData || {}));
+        } catch (err) {
+          console.log('There was an error parsing JSON for ' + pattern.abspath);
+          console.log(err);
         }
 
         var allData = pattern_assembler.merge_data(globalData, localData);