fix: Allow using of non-word characters in [import] tag (#52)

* Allow more characters to be used in string fields of [include] tag

* Removed unwanted file.

* Respond to comments in pull request #68

* Fix errors in test, fix typo, add license for unescape-string.js

* Update dependency of gitbook-cli in example builds

* Respond to comments again. (Did not read a comment correctly)

* Fix missing empty line

* Changed the last assert.equal to assert.strictEqual

Author: lwchkg

LICENSE

@@ -17,3 +17,34 @@ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
 LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
 OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
 SOFTWARE.
+
+---
+
+src/unescape-string.js is extracted from https://github.com/jgm/commonmark.js:
+
+Copyright (c) 2014, John MacFarlane
+
+All rights reserved.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions are met:
+
+    * Redistributions of source code must retain the above copyright
+      notice, this list of conditions and the following disclaimer.
+
+    * Redistributions in binary form must reproduce the above
+      copyright notice, this list of conditions and the following
+      disclaimer in the documentation and/or other materials provided
+      with the distribution.
+
+THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
+"AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
+LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
+A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
+OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
+SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
+LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
+DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
+THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
+(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
+OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

examples/ace/package.json

@@ -15,7 +15,7 @@
   "author": "azu",
   "license": "MIT",
   "devDependencies": {
-    "gitbook-cli": "^2.1.3",
+    "gitbook-cli": "2.3.0 || ^2.3.2",
     "gitbook-plugin-include-codeblock": "file:../../"
   }
 }

examples/custom/package.json

@@ -15,7 +15,7 @@
   "author": "azu",
   "license": "MIT",
   "devDependencies": {
-    "gitbook-cli": "^2.1.3",
+    "gitbook-cli": "2.3.0 || ^2.3.2",
     "gitbook-plugin-include-codeblock": "file:../../"
   }
 }

examples/default/package.json

@@ -15,7 +15,7 @@
   "author": "azu",
   "license": "MIT",
   "devDependencies": {
-    "gitbook-cli": "^2.1.3",
+    "gitbook-cli": "2.3.0 || ^2.3.2",
     "gitbook-plugin-include-codeblock": "file:../../"
   }
 }

package.json

@@ -89,6 +89,7 @@
     }
   },
   "dependencies": {
+    "entities": "^1.1.1",
     "language-map": "^1.1.1",
     "handlebars": "^4.0.5",
     "winston-color": "^1.0.0"

src/parser.js

@@ -3,13 +3,25 @@
 const path = require("path");
 const Handlebars = require("handlebars");
 const logger = require("winston-color");
-import { defaultKeyValueMap, initOptions, checkMapTypes, convertValue } from "./options.js";
+import { defaultKeyValueMap, initOptions, checkMapTypes } from "./options.js";
+import { unescapeString } from "./unescape-string.js";
 import { getLang } from "./language-detection";
 import { getMarker, hasMarker, markerSliceCode, removeMarkers } from "./marker";
 import { sliceCode, hasSliceRange, getSliceRange } from "./slicer";
 import { hasTitle } from "./title";
 import { getTemplateContent, readFileFromPath } from "./template";
-const markdownLinkFormatRegExp = /\[([^\]]*?)\]\(([^\)]*?)\)/gm;
+const markdownLinkFormatRegExp = /\[((?:[^\]]|\\.)*?)\]\(((?:[^\)]|\\.)*?)\)/gm;
+
+const keyEx = "\\w+";
+const kvsepEx = "[:=]";
+const spacesEx = "\\s*";
+const quoteEx = "[\"']";
+const valEx = "(?:[^'\"\\\\]|\\\\.)*";
+const argEx = `${quoteEx}${valEx}${quoteEx}|true|false`;
+const expressionEx = `(${keyEx})${kvsepEx}${spacesEx}(${argEx})`;
+const expressionRegExp = new RegExp(expressionEx, "g");
+
+const markerRegExp = /^\s*(([-\w\s]*,?)*)$/;
 
 /**
  * A counter to count how many code are imported.
@@ -68,6 +80,48 @@ export function containIncludeCommand(commands = []) {
     });
 }
 
+/**
+ * Parse the given value to the given type. Returns the value if valid, otherwise returns undefined.
+ * @param {string} value
+ * @param {string} type "string", "boolean"
+ * @param {string} key
+ * @return {boolean|string|undefined}
+ */
+export function parseValue(value, type, key) {
+    if (type === "string") {
+        const unescapedvalue = unescapeString(value.substring(1, value.length - 1));
+        if (key === "marker" && !markerRegExp.test(unescapedvalue)) {
+            logger.error(
+                "include-codeblock: parseVariablesFromLabel: invalid value " +
+                    `\`${unescapedvalue}\` in key \`marker\``
+            );
+            return undefined;
+        }
+        return unescapedvalue;
+    }
+
+    if (type === "boolean") {
+        if (["true", '"true"', "'true'"].indexOf(value) >= 0) {
+            return true;
+        }
+
+        if (["false", '"false"', "'false'"].indexOf(value) >= 0) {
+            return false;
+        }
+
+        logger.error(
+            "include-codeblock: parseVariablesFromLabel: invalid value " +
+                `\`${value}\` in key \`${key}\`. Expect true or false.`
+        );
+        return undefined;
+    }
+
+    logger.error(
+        `include-codeblock: parseVariablesFromLabel: unknown key type \`${type}\` (see options.js)`
+    );
+    return undefined;
+}
+
 /** Parse the command label and return a new key-value object
  * @example
  *      [import,title:"<thetitle>",label:"<thelabel>"](path/to/file.ext)
@@ -77,56 +131,29 @@ export function containIncludeCommand(commands = []) {
  */
 export function parseVariablesFromLabel(kvMap, label) {
     const kv = Object.assign({}, kvMap);
-    const beginEx = "^.*";
-    const endEx = ".*$";
-    const sepEx = ",?";
-    const kvsepEx = "[:=]";
-    const spacesEx = "\\s*";
-    const quotesEx = "[\"']";
-
-    Object.keys(kv).forEach(key => {
-        let keyEx = "(" + key + ")";
-        let valEx = "([-\\w\\s]*)";
-        if (key === "marker") {
-            keyEx = "(import|include)";
-            valEx = "(([-\\w\\s]*,?)*)";
+
+    let match = "";
+    while ((match = expressionRegExp.exec(label))) {
+        let key = match[1];
+        if (key === "include" || key === "import") {
+            key = "marker";
         }
-        // Add value check here
-        switch (typeof defaultKeyValueMap[key]) {
-            case "string":
-                valEx = quotesEx + valEx + quotesEx;
-                break;
-            case "boolean":
-                // no quotes
-                valEx = quotesEx + "?(true|false)" + quotesEx + "?";
-                break;
-            default:
-                logger.error(
-                    "include-codeblock: parseVariablesFromLabel: key type `" +
-                        typeof defaultKeyValueMap[key] +
-                        "` unknown (see options.js)"
-                );
-                break;
+        const value = match[2];
+
+        if (!kv.hasOwnProperty(key)) {
+            logger.error(
+                "include-codeblock: parseVariablesFromLabel: unknown key " +
+                    `\`${key}\` (see options.js)`
+            );
+            return;
         }
-        // Val type cast to string.
-        const regStr =
-            beginEx +
-            sepEx +
-            spacesEx +
-            keyEx +
-            spacesEx +
-            kvsepEx +
-            spacesEx +
-            valEx +
-            spacesEx +
-            sepEx +
-            endEx;
-        const reg = new RegExp(regStr);
-        const res = label.match(reg);
-        if (res) {
-            kv[key] = convertValue(res[2], typeof defaultKeyValueMap[key]);
+
+        const parsedValue = parseValue(value, typeof defaultKeyValueMap[key], key);
+        if (parsedValue !== undefined) {
+            kv[key] = parsedValue;
         }
-    });
+    }
+
     return Object.freeze(kv);
 }
 

src/unescape-string.js

@@ -0,0 +1,29 @@
+// The code in this file is extracted from commonmark.js
+// (https://github.com/jgm/commonmark.js), which is owned by John MacFarlane.
+// LICENSE : BSD-2-Clause
+"use strict";
+
+var decodeHTML = require("entities").decodeHTML;
+
+var C_BACKSLASH = 92;
+var ENTITY = "&(?:#x[a-f0-9]{1,8}|#[0-9]{1,8}|[a-z][a-z0-9]{1,31});";
+var reBackslashOrAmp = /[\\&]/;
+var ESCAPABLE = "[!\"#$%&'()*+,./:;<=>?@[\\\\\\]^_`{|}~-]";
+var reEntityOrEscapedChar = new RegExp("\\\\" + ESCAPABLE + "|" + ENTITY, "gi");
+
+function unescapeChar(s) {
+    if (s.charCodeAt(0) === C_BACKSLASH) {
+        return s.charAt(1);
+    } else {
+        return decodeHTML(s);
+    }
+}
+
+// Replace entities and backslash escapes with literal characters.
+export function unescapeString(s) {
+    if (reBackslashOrAmp.test(s)) {
+        return s.replace(reEntityOrEscapedChar, unescapeChar);
+    } else {
+        return s;
+    }
+}

test/parser-test.js

@@ -6,6 +6,7 @@ import {
     containIncludeCommand,
     splitLabelToCommands,
     strip,
+    parseValue,
     parseVariablesFromLabel
 } from "../src/parser";
 
@@ -45,7 +46,45 @@ describe("parse", function() {
             assert(containIncludeCommand(commands));
         });
     });
-    context("parseVariablesFromLabel ", function() {
+    describe("parseValue", function() {
+        it("should unescape string parameter", function() {
+            const result = parseValue(
+                '"\\!\\"\\#\\$\\%\\&\\\'\\(\\)\\*\\+\\,\\-\\.\\/' +
+                    "\\:\\;\\<\\=\\>\\?\\@\\[\\\\\\]\\^\\_\\`\\{\\|\\}\\~" +
+                    '&amp;&lt;&gt;&#65;&#x41;"',
+                "string",
+                ""
+            );
+            assert.strictEqual(result, "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~&<>AA");
+        });
+        it("should backslash unescape commonmark defined characters only", function() {
+            const result = parseValue('"\\r\\n\\[\\]"', "string", "");
+            assert.strictEqual(result, "\\r\\n[]"); // \r and \n should not be unescaped.
+        });
+        it("should validate markers", function() {
+            let result = parseValue('" marker0 , marker1 "', "string", "marker");
+            assert.strictEqual(result, " marker0 , marker1 ");
+
+            result = parseValue('"~invalid~"', "string", "marker");
+            assert.strictEqual(result, undefined);
+        });
+        it("should parse boolean values", function() {
+            let result = parseValue("true", "boolean", "");
+            assert.strictEqual(result, true);
+            result = parseValue('"true"', "boolean", "");
+            assert.strictEqual(result, true);
+            result = parseValue("'true'", "boolean", "");
+            assert.strictEqual(result, true);
+
+            result = parseValue("false", "boolean", "");
+            assert.strictEqual(result, false);
+            result = parseValue('"false"', "boolean", "");
+            assert.strictEqual(result, false);
+            result = parseValue("'false'", "boolean", "");
+            assert.strictEqual(result, false);
+        });
+    });
+    describe("parseVariablesFromLabel ", function() {
         it("should retrieve edit boolean", function() {
             const resmap = parseVariablesFromLabel(kvmap, "include,edit:true");
             const results = resmap;
@@ -113,6 +152,36 @@ describe("parse", function() {
             const results = resmap;
             assert.equal(results.marker, "");
         });
+        it("should handle characters for string parameter", function() {
+            const resmap = parseVariablesFromLabel(
+                kvmap,
+                'import,title="test+with-special*string"'
+            );
+            const results = resmap;
+            assert.strictEqual(results.title, "test+with-special*string");
+        });
+        it("should unescape string parameter", function() {
+            const resmap = parseVariablesFromLabel(
+                kvmap,
+                'import,title="\\!\\"\\#\\$\\%\\&\\\'\\(\\)\\*\\+\\,\\-\\.\\/' +
+                    "\\:\\;\\<\\=\\>\\?\\@\\[\\\\\\]\\^\\_\\`\\{\\|\\}\\~" +
+                    '&amp;&lt;&gt;&#65;&#x41;"'
+            );
+            const results = resmap;
+            assert.strictEqual(results.title, "!\"#$%&'()*+,-./:;<=>?@[\\]^_`{|}~&<>AA");
+        });
+        it("should backslash unescape commonmark defined characters only", function() {
+            const resmap = parseVariablesFromLabel(kvmap, 'import,title="\\r\\n\\[\\]"');
+            const results = resmap;
+            assert.strictEqual(results.title, "\\r\\n[]"); // \r and \n should not be unescaped.
+        });
+        it("should not parse string argument into another key-value pair", function() {
+            assert.strictEqual(kvmap.edit, false);
+            const resmap = parseVariablesFromLabel(kvmap, 'import,title="edit:true"');
+            const results = resmap;
+            assert.strictEqual(results.edit, false);
+            assert.strictEqual(results.title, "edit:true");
+        });
     });
     // inspired from https://github.com/rails/rails/blob/master/activesupport/test/core_ext/string_ext_test.rb
     describe("strip", function() {

test/patterns/import-output-escape/actual.md

@@ -0,0 +1 @@
+[import, title:"title<1>", id:"edit:true<2>", lang:"javascript+theme:abc<3>", class="class<4>", edit=true, check=true, theme="monokai"](./test&1.js)

test/patterns/import-output-escape/book.js

@@ -0,0 +1,8 @@
+const path = require("path");
+module.exports = {
+    "pluginsConfig": {
+        "include-codeblock": {
+            "template": path.join(__dirname, "dump.hbs")
+        }
+    }
+};