feat: add raw content token & node

src/constants/node-types.ts

@@ -3,6 +3,7 @@ export enum NodeTypes {
   Tag = "Tag",
   Text = "Text",
   Doctype = "Doctype",
+  RawContent = "RawContent",
 
   Comment = "Comment",
   CommentOpen = "CommentOpen",

src/constants/token-types.ts

@@ -1,5 +1,6 @@
 export enum TokenTypes {
   Text = "Text",
+  RawContent = "RawContent",
   OpenTagStart = "OpenTagStart",
   OpenTagEnd = "OpenTagEnd",
   CloseTag = "CloseTag",

src/index.ts

@@ -1,4 +1,5 @@
 export { parse } from "./parser";
 export { NodeTypes, TokenTypes } from "./constants";
 export { AnyToken, ParseResult } from "./types";
+export { Options } from "./types/parse";
 export * from "./types/node";

src/tokenizer/__tests__/__output__/custom-tag-raw-content.ts

@@ -110,7 +110,7 @@ const OUTPUT: AnyToken[] = [
     },
   },
   {
-    type: TokenTypes.Text,
+    type: TokenTypes.RawContent,
     value: `
 # Hello, world!
 

src/tokenizer/handlers/custom-tag-raw-content.ts

@@ -35,11 +35,11 @@ function parseClosingCustomTag(state: TokenizerState) {
   if (state.accumulatedContent.value() !== "") {
     const position = calculateTokenPosition(state, { keepBuffer: false });
     state.tokens.push({
-      type: TokenTypes.Text,
+      type: TokenTypes.RawContent,
       value: state.accumulatedContent.value(),
       range: position.range,
       loc: position.loc,
-      parts: createParts(state, TokenTypes.Text),
+      parts: createParts(state, TokenTypes.RawContent),
     });
   }
 

src/tree-constructor/__tests__/__file_snapshots__/construct-tree-Custom-Tag-Raw-Content-0.json

@@ -0,0 +1,206 @@
+{
+  "type": "Document",
+  "range": [
+    0,
+    271
+  ],
+  "children": [
+    {
+      "type": "Tag",
+      "range": [
+        0,
+        271
+      ],
+      "loc": {
+        "start": {
+          "line": 1,
+          "column": 0
+        },
+        "end": {
+          "line": 17,
+          "column": 11
+        }
+      },
+      "attributes": [
+        {
+          "type": "Attribute",
+          "range": [
+            10,
+            18
+          ],
+          "loc": {
+            "start": {
+              "line": 1,
+              "column": 10
+            },
+            "end": {
+              "line": 1,
+              "column": 18
+            }
+          },
+          "key": {
+            "type": "AttributeKey",
+            "value": "attr",
+            "loc": {
+              "start": {
+                "line": 1,
+                "column": 10
+              },
+              "end": {
+                "line": 1,
+                "column": 14
+              }
+            },
+            "range": [
+              10,
+              14
+            ],
+            "parts": []
+          },
+          "startWrapper": {
+            "type": "AttributeValueWrapperStart",
+            "value": "\"",
+            "loc": {
+              "start": {
+                "line": 1,
+                "column": 15
+              },
+              "end": {
+                "line": 1,
+                "column": 16
+              }
+            },
+            "range": [
+              15,
+              16
+            ]
+          },
+          "value": {
+            "type": "AttributeValue",
+            "value": "1",
+            "loc": {
+              "start": {
+                "line": 1,
+                "column": 16
+              },
+              "end": {
+                "line": 1,
+                "column": 17
+              }
+            },
+            "range": [
+              16,
+              17
+            ],
+            "parts": []
+          },
+          "endWrapper": {
+            "type": "AttributeValueWrapperEnd",
+            "value": "\"",
+            "loc": {
+              "start": {
+                "line": 1,
+                "column": 17
+              },
+              "end": {
+                "line": 1,
+                "column": 18
+              }
+            },
+            "range": [
+              17,
+              18
+            ]
+          }
+        }
+      ],
+      "children": [
+        {
+          "type": "RawContent",
+          "value": "\n# Hello, world!\n\n```cpp{4-6,9}\n#include <iostream>\n\nclass Example {\n    Example() {\n        std::cout << \"Hello, world!\" << std::endl;\n    }\n\n    Example(std::string name) {\n        std::cout << \"Hello, \" << name << std::endl;\n    }\n};\n```\n",
+          "loc": {
+            "start": {
+              "line": 1,
+              "column": 19
+            },
+            "end": {
+              "line": 17,
+              "column": 0
+            }
+          },
+          "range": [
+            19,
+            260
+          ],
+          "parts": []
+        }
+      ],
+      "openStart": {
+        "type": "OpenTagStart",
+        "value": "<markdown",
+        "loc": {
+          "start": {
+            "line": 1,
+            "column": 0
+          },
+          "end": {
+            "line": 1,
+            "column": 9
+          }
+        },
+        "range": [
+          0,
+          9
+        ]
+      },
+      "name": "markdown",
+      "openEnd": {
+        "type": "OpenTagEnd",
+        "value": ">",
+        "loc": {
+          "start": {
+            "line": 1,
+            "column": 18
+          },
+          "end": {
+            "line": 1,
+            "column": 19
+          }
+        },
+        "range": [
+          18,
+          19
+        ]
+      },
+      "selfClosing": false,
+      "close": {
+        "type": "CloseTag",
+        "value": "</markdown>",
+        "loc": {
+          "start": {
+            "line": 17,
+            "column": 0
+          },
+          "end": {
+            "line": 17,
+            "column": 11
+          }
+        },
+        "range": [
+          260,
+          271
+        ]
+      }
+    }
+  ],
+  "loc": {
+    "start": {
+      "line": 1,
+      "column": 0
+    },
+    "end": {
+      "line": 17,
+      "column": 11
+    }
+  }
+}

src/tree-constructor/__tests__/construct-tree.spec.ts

@@ -43,6 +43,7 @@ import TEMPLATE_COMMENT from "../../tokenizer/__tests__/__output__/templates-com
 import TEMPLATE_SCRIPT_CONTENT from "../../tokenizer/__tests__/__output__/templates-script-content";
 import TEMPLATE_STYLE_CONTENT from "../../tokenizer/__tests__/__output__/templates-style-content";
 import TEMPLATE_CONTENT_END from "../../tokenizer/__tests__/__output__/templates-content-end";
+import CUSTOM_TAG_RAW_CONTENT from "../../tokenizer/__tests__/__output__/custom-tag-raw-content";
 import { clearParent } from "../../utils";
 import { toMatchFile } from "jest-file-snapshot";
 
@@ -78,6 +79,7 @@ describe("construct-tree", () => {
     ["Templates Script Content", TEMPLATE_SCRIPT_CONTENT],
     ["Templates Style Content", TEMPLATE_STYLE_CONTENT],
     ["Templates Content End", TEMPLATE_CONTENT_END],
+    ["Custom Tag Raw Content", CUSTOM_TAG_RAW_CONTENT],
   ])("%s", (name: string, inputTokens: any) => {
     const { ast } = constructTree(inputTokens);
     expect(JSON.stringify(clearParent(ast), null, 2)).toMatchFile(undefined, {

src/tree-constructor/handlers/tag-content.ts

@@ -15,6 +15,7 @@ import {
   ContextualDoctypeNode,
   TextNode,
   CompositeToken,
+  RawContentNode,
 } from "../../types";
 import {
   cloneRange,
@@ -131,6 +132,19 @@ function handleText(
   return state;
 }
 
+function handleRawContent(
+  state: ConstructTreeState<ContextualTagNode | ContextualDocumentNode>,
+  token: CompositeToken<TokenTypes.RawContent>
+) {
+  initChildrenIfNone(state.currentNode);
+  const rawContentNode = createNodeFrom(token) as RawContentNode;
+
+  state.currentNode.children.push(rawContentNode);
+  state.caretPosition++;
+
+  return state;
+}
+
 function handleOpenScriptTagStart(
   state: ConstructTreeState<ContextualTagNode | ContextualDocumentNode>,
   token: Token<TokenTypes.OpenScriptTagStart>
@@ -202,6 +216,10 @@ export function construct(
     return handleText(state, token);
   }
 
+  if (token.type === TokenTypes.RawContent) {
+    return handleRawContent(state, token);
+  }
+
   if (token.type === TokenTypes.CloseTag) {
     return handleCloseTag(state, token);
   }

src/types/node.ts

@@ -11,6 +11,7 @@ export interface DocumentNode extends BaseNode {
   type: NodeTypes.Document;
   children: Array<
     | TextNode
+    | RawContentNode
     | TagNode
     | ScriptTagNode
     | StyleTagNode
@@ -20,6 +21,7 @@ export interface DocumentNode extends BaseNode {
 }
 
 export type TextNode = CompositeNode<NodeTypes.Text>;
+export type RawContentNode = CompositeNode<NodeTypes.RawContent>;
 
 export interface TagNode extends BaseNode {
   type: NodeTypes.Tag;
@@ -29,7 +31,12 @@ export interface TagNode extends BaseNode {
   openEnd: OpenTagEndNode;
   close?: CloseTagNode;
   children: Array<
-    TextNode | TagNode | ScriptTagNode | StyleTagNode | CommentNode
+    | TextNode
+    | TagNode
+    | ScriptTagNode
+    | StyleTagNode
+    | CommentNode
+    | RawContentNode
   >;
   attributes: Array<AttributeNode>;
 }

src/types/token.ts

@@ -30,6 +30,7 @@ export interface CompositeToken<T extends TokenTypes> extends Token<T> {
 
 export type AnyToken =
   | CompositeToken<TokenTypes.Text>
+  | CompositeToken<TokenTypes.RawContent>
   | Token<TokenTypes.OpenTagStart>
   | Token<TokenTypes.OpenTagEnd>
   | Token<TokenTypes.CloseTag>