From 014baf9e05b94256b31001a8518c14142052e4d6 Mon Sep 17 00:00:00 2001 From: YeonJuan Date: Tue, 29 Jul 2025 22:27:34 +0900 Subject: [PATCH] feat: add raw content token & node --- src/constants/node-types.ts | 1 + src/constants/token-types.ts | 1 + src/index.ts | 1 + .../__output__/custom-tag-raw-content.ts | 2 +- .../handlers/custom-tag-raw-content.ts | 4 +- ...nstruct-tree-Custom-Tag-Raw-Content-0.json | 206 ++++++++++++++++++ .../__tests__/construct-tree.spec.ts | 2 + src/tree-constructor/handlers/tag-content.ts | 18 ++ src/types/node.ts | 9 +- src/types/token.ts | 1 + 10 files changed, 241 insertions(+), 4 deletions(-) create mode 100644 src/tree-constructor/__tests__/__file_snapshots__/construct-tree-Custom-Tag-Raw-Content-0.json diff --git a/src/constants/node-types.ts b/src/constants/node-types.ts index 216672b..1da6e72 100644 --- a/src/constants/node-types.ts +++ b/src/constants/node-types.ts @@ -3,6 +3,7 @@ export enum NodeTypes { Tag = "Tag", Text = "Text", Doctype = "Doctype", + RawContent = "RawContent", Comment = "Comment", CommentOpen = "CommentOpen", diff --git a/src/constants/token-types.ts b/src/constants/token-types.ts index c5e090d..c41f263 100644 --- a/src/constants/token-types.ts +++ b/src/constants/token-types.ts @@ -1,5 +1,6 @@ export enum TokenTypes { Text = "Text", + RawContent = "RawContent", OpenTagStart = "OpenTagStart", OpenTagEnd = "OpenTagEnd", CloseTag = "CloseTag", diff --git a/src/index.ts b/src/index.ts index 6270ba6..3394623 100644 --- a/src/index.ts +++ b/src/index.ts @@ -1,4 +1,5 @@ export { parse } from "./parser"; export { NodeTypes, TokenTypes } from "./constants"; export { AnyToken, ParseResult } from "./types"; +export { Options } from "./types/parse"; export * from "./types/node"; diff --git a/src/tokenizer/__tests__/__output__/custom-tag-raw-content.ts b/src/tokenizer/__tests__/__output__/custom-tag-raw-content.ts index 6264a5b..3bd1882 100644 --- a/src/tokenizer/__tests__/__output__/custom-tag-raw-content.ts +++ b/src/tokenizer/__tests__/__output__/custom-tag-raw-content.ts @@ -110,7 +110,7 @@ const OUTPUT: AnyToken[] = [ }, }, { - type: TokenTypes.Text, + type: TokenTypes.RawContent, value: ` # Hello, world! diff --git a/src/tokenizer/handlers/custom-tag-raw-content.ts b/src/tokenizer/handlers/custom-tag-raw-content.ts index 9ad3dea..8c3035c 100644 --- a/src/tokenizer/handlers/custom-tag-raw-content.ts +++ b/src/tokenizer/handlers/custom-tag-raw-content.ts @@ -35,11 +35,11 @@ function parseClosingCustomTag(state: TokenizerState) { if (state.accumulatedContent.value() !== "") { const position = calculateTokenPosition(state, { keepBuffer: false }); state.tokens.push({ - type: TokenTypes.Text, + type: TokenTypes.RawContent, value: state.accumulatedContent.value(), range: position.range, loc: position.loc, - parts: createParts(state, TokenTypes.Text), + parts: createParts(state, TokenTypes.RawContent), }); } diff --git a/src/tree-constructor/__tests__/__file_snapshots__/construct-tree-Custom-Tag-Raw-Content-0.json b/src/tree-constructor/__tests__/__file_snapshots__/construct-tree-Custom-Tag-Raw-Content-0.json new file mode 100644 index 0000000..6ca7ff3 --- /dev/null +++ b/src/tree-constructor/__tests__/__file_snapshots__/construct-tree-Custom-Tag-Raw-Content-0.json @@ -0,0 +1,206 @@ +{ + "type": "Document", + "range": [ + 0, + 271 + ], + "children": [ + { + "type": "Tag", + "range": [ + 0, + 271 + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 17, + "column": 11 + } + }, + "attributes": [ + { + "type": "Attribute", + "range": [ + 10, + 18 + ], + "loc": { + "start": { + "line": 1, + "column": 10 + }, + "end": { + "line": 1, + "column": 18 + } + }, + "key": { + "type": "AttributeKey", + "value": "attr", + "loc": { + "start": { + "line": 1, + "column": 10 + }, + "end": { + "line": 1, + "column": 14 + } + }, + "range": [ + 10, + 14 + ], + "parts": [] + }, + "startWrapper": { + "type": "AttributeValueWrapperStart", + "value": "\"", + "loc": { + "start": { + "line": 1, + "column": 15 + }, + "end": { + "line": 1, + "column": 16 + } + }, + "range": [ + 15, + 16 + ] + }, + "value": { + "type": "AttributeValue", + "value": "1", + "loc": { + "start": { + "line": 1, + "column": 16 + }, + "end": { + "line": 1, + "column": 17 + } + }, + "range": [ + 16, + 17 + ], + "parts": [] + }, + "endWrapper": { + "type": "AttributeValueWrapperEnd", + "value": "\"", + "loc": { + "start": { + "line": 1, + "column": 17 + }, + "end": { + "line": 1, + "column": 18 + } + }, + "range": [ + 17, + 18 + ] + } + } + ], + "children": [ + { + "type": "RawContent", + "value": "\n# Hello, world!\n\n```cpp{4-6,9}\n#include \n\nclass Example {\n Example() {\n std::cout << \"Hello, world!\" << std::endl;\n }\n\n Example(std::string name) {\n std::cout << \"Hello, \" << name << std::endl;\n }\n};\n```\n", + "loc": { + "start": { + "line": 1, + "column": 19 + }, + "end": { + "line": 17, + "column": 0 + } + }, + "range": [ + 19, + 260 + ], + "parts": [] + } + ], + "openStart": { + "type": "OpenTagStart", + "value": "", + "loc": { + "start": { + "line": 1, + "column": 18 + }, + "end": { + "line": 1, + "column": 19 + } + }, + "range": [ + 18, + 19 + ] + }, + "selfClosing": false, + "close": { + "type": "CloseTag", + "value": "", + "loc": { + "start": { + "line": 17, + "column": 0 + }, + "end": { + "line": 17, + "column": 11 + } + }, + "range": [ + 260, + 271 + ] + } + } + ], + "loc": { + "start": { + "line": 1, + "column": 0 + }, + "end": { + "line": 17, + "column": 11 + } + } +} \ No newline at end of file diff --git a/src/tree-constructor/__tests__/construct-tree.spec.ts b/src/tree-constructor/__tests__/construct-tree.spec.ts index 49b34cd..851dfa6 100644 --- a/src/tree-constructor/__tests__/construct-tree.spec.ts +++ b/src/tree-constructor/__tests__/construct-tree.spec.ts @@ -43,6 +43,7 @@ import TEMPLATE_COMMENT from "../../tokenizer/__tests__/__output__/templates-com import TEMPLATE_SCRIPT_CONTENT from "../../tokenizer/__tests__/__output__/templates-script-content"; import TEMPLATE_STYLE_CONTENT from "../../tokenizer/__tests__/__output__/templates-style-content"; import TEMPLATE_CONTENT_END from "../../tokenizer/__tests__/__output__/templates-content-end"; +import CUSTOM_TAG_RAW_CONTENT from "../../tokenizer/__tests__/__output__/custom-tag-raw-content"; import { clearParent } from "../../utils"; import { toMatchFile } from "jest-file-snapshot"; @@ -78,6 +79,7 @@ describe("construct-tree", () => { ["Templates Script Content", TEMPLATE_SCRIPT_CONTENT], ["Templates Style Content", TEMPLATE_STYLE_CONTENT], ["Templates Content End", TEMPLATE_CONTENT_END], + ["Custom Tag Raw Content", CUSTOM_TAG_RAW_CONTENT], ])("%s", (name: string, inputTokens: any) => { const { ast } = constructTree(inputTokens); expect(JSON.stringify(clearParent(ast), null, 2)).toMatchFile(undefined, { diff --git a/src/tree-constructor/handlers/tag-content.ts b/src/tree-constructor/handlers/tag-content.ts index de150cf..409a7ef 100644 --- a/src/tree-constructor/handlers/tag-content.ts +++ b/src/tree-constructor/handlers/tag-content.ts @@ -15,6 +15,7 @@ import { ContextualDoctypeNode, TextNode, CompositeToken, + RawContentNode, } from "../../types"; import { cloneRange, @@ -131,6 +132,19 @@ function handleText( return state; } +function handleRawContent( + state: ConstructTreeState, + token: CompositeToken +) { + initChildrenIfNone(state.currentNode); + const rawContentNode = createNodeFrom(token) as RawContentNode; + + state.currentNode.children.push(rawContentNode); + state.caretPosition++; + + return state; +} + function handleOpenScriptTagStart( state: ConstructTreeState, token: Token @@ -202,6 +216,10 @@ export function construct( return handleText(state, token); } + if (token.type === TokenTypes.RawContent) { + return handleRawContent(state, token); + } + if (token.type === TokenTypes.CloseTag) { return handleCloseTag(state, token); } diff --git a/src/types/node.ts b/src/types/node.ts index e55f88b..7f99c95 100644 --- a/src/types/node.ts +++ b/src/types/node.ts @@ -11,6 +11,7 @@ export interface DocumentNode extends BaseNode { type: NodeTypes.Document; children: Array< | TextNode + | RawContentNode | TagNode | ScriptTagNode | StyleTagNode @@ -20,6 +21,7 @@ export interface DocumentNode extends BaseNode { } export type TextNode = CompositeNode; +export type RawContentNode = CompositeNode; export interface TagNode extends BaseNode { type: NodeTypes.Tag; @@ -29,7 +31,12 @@ export interface TagNode extends BaseNode { openEnd: OpenTagEndNode; close?: CloseTagNode; children: Array< - TextNode | TagNode | ScriptTagNode | StyleTagNode | CommentNode + | TextNode + | TagNode + | ScriptTagNode + | StyleTagNode + | CommentNode + | RawContentNode >; attributes: Array; } diff --git a/src/types/token.ts b/src/types/token.ts index 1f04927..b712a8a 100644 --- a/src/types/token.ts +++ b/src/types/token.ts @@ -30,6 +30,7 @@ export interface CompositeToken extends Token { export type AnyToken = | CompositeToken + | CompositeToken | Token | Token | Token