allowComments in next.js plugin (#24)

* allowComments in next.js plugin

* pass as tokenizerOptions

Author: mfix-stripe

src/loader.js

@@ -8,7 +8,7 @@ function normalize(s) {
   return s.replace(/\\/g, path.win32.sep.repeat(2));
 }
 
-async function gatherPartials(ast, schemaDir) {
+async function gatherPartials(ast, schemaDir, tokenizer) {
   let partials = {};
 
   for (const node of ast.walk()) {
@@ -25,11 +25,12 @@ async function gatherPartials(ast, schemaDir) {
       const content = await fs.promises.readFile(filepath, {encoding: 'utf8'});
 
       if (content) {
-        const ast = Markdoc.parse(content);
+        const tokens = tokenizer.tokenize(content);
+        const ast = Markdoc.parse(tokens);
         partials = {
           ...partials,
           [file]: content,
-          ...(await gatherPartials.call(this, ast, schemaDir)),
+          ...(await gatherPartials.call(this, ast, schemaDir, tokenizer)),
         };
       }
     }
@@ -51,10 +52,14 @@ async function load(source) {
     dir, // Root directory from Next.js (contains next.config.js)
     mode = 'static',
     schemaPath = DEFAULT_SCHEMA_PATH,
+    tokenizerOptions = undefined,
   } = this.getOptions() || {};
 
+  const tokenizer = new Markdoc.Tokenizer(tokenizerOptions);
+
   const schemaDir = path.resolve(dir, schemaPath || DEFAULT_SCHEMA_PATH);
-  const ast = Markdoc.parse(source);
+  const tokens = tokenizer.tokenize(source);
+  const ast = Markdoc.parse(tokens);
 
   // Grabs the path of the file relative to the `/pages` directory
   // to pass into the app props later.
@@ -66,7 +71,8 @@ async function load(source) {
   const partials = await gatherPartials.call(
     this,
     ast,
-    path.resolve(schemaDir, 'partials')
+    path.resolve(schemaDir, 'partials'),
+    tokenizer
   );
 
   // IDEA: consider making this an option per-page
@@ -124,12 +130,17 @@ import {getSchema, defaultObject} from '${normalize(
  */
 ${schemaCode}
 
+const tokenizer = new Markdoc.Tokenizer(${
+    tokenizerOptions ? JSON.stringify(tokenizerOptions) : ''
+  });
+
 /**
  * Source will never change at runtime, so parse happens at the file root
  */
 const source = ${JSON.stringify(source)};
 const filepath = ${JSON.stringify(filepath)};
-const ast = Markdoc.parse(source);
+const tokens = tokenizer.tokenize(source);
+const ast = Markdoc.parse(tokens);
 
 /**
  * Like the AST, frontmatter won't change at runtime, so it is loaded at file root.
@@ -146,7 +157,8 @@ export async function ${dataFetchingFunction}(context) {
 
   // Ensure Node.transformChildren is available
   Object.keys(partials).forEach((key) => {
-    partials[key] = Markdoc.parse(partials[key]);
+    const tokens = tokenizer.tokenize(partials[key]);
+    partials[key] = Markdoc.parse(tokens);
   });
 
   const cfg = {

tests/__snapshots__/index.test.js.snap

@@ -13,12 +13,15 @@ import {getSchema, defaultObject} from './src/runtime.js';
  */
 const schema = {};
 
+const tokenizer = new Markdoc.Tokenizer();
+
 /**
  * Source will never change at runtime, so parse happens at the file root
  */
 const source = \\"---\\\\ntitle: Custom title\\\\n---\\\\n\\\\n# {% $markdoc.frontmatter.title %}\\\\n\\\\n{% tag /%}\\\\n\\";
 const filepath = \\"/test/index.md\\";
-const ast = Markdoc.parse(source);
+const tokens = tokenizer.tokenize(source);
+const ast = Markdoc.parse(tokens);
 
 /**
  * Like the AST, frontmatter won't change at runtime, so it is loaded at file root.
@@ -35,7 +38,8 @@ export async function getStaticProps(context) {
 
   // Ensure Node.transformChildren is available
   Object.keys(partials).forEach((key) => {
-    partials[key] = Markdoc.parse(partials[key]);
+    const tokens = tokenizer.tokenize(partials[key]);
+    partials[key] = Markdoc.parse(tokens);
   });
 
   const cfg = {
@@ -99,12 +103,15 @@ import {getSchema, defaultObject} from './src/runtime.js';
  */
 const schema = {};
 
+const tokenizer = new Markdoc.Tokenizer();
+
 /**
  * Source will never change at runtime, so parse happens at the file root
  */
 const source = \\"---\\\\ntitle: Custom title\\\\n---\\\\n\\\\n# {% $markdoc.frontmatter.title %}\\\\n\\\\n{% tag /%}\\\\n\\";
 const filepath = \\"/test/index.md\\";
-const ast = Markdoc.parse(source);
+const tokens = tokenizer.tokenize(source);
+const ast = Markdoc.parse(tokens);
 
 /**
  * Like the AST, frontmatter won't change at runtime, so it is loaded at file root.
@@ -121,7 +128,8 @@ export async function getStaticProps(context) {
 
   // Ensure Node.transformChildren is available
   Object.keys(partials).forEach((key) => {
-    partials[key] = Markdoc.parse(partials[key]);
+    const tokens = tokenizer.tokenize(partials[key]);
+    partials[key] = Markdoc.parse(tokens);
   });
 
   const cfg = {