Skip to content

Commit 3f2104b

Browse files
yeonjuanyeonjuan
authored
feat: add raw content token & node (#33)
1 parent ab87f37 commit 3f2104b

File tree

10 files changed

+241
-4
lines changed

10 files changed

+241
-4
lines changed

‎src/constants/node-types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,7 @@ export enum NodeTypes {
33
Tag = "Tag",
44
Text = "Text",
55
Doctype = "Doctype",
6+
RawContent = "RawContent",
67

78
Comment = "Comment",
89
CommentOpen = "CommentOpen",

‎src/constants/token-types.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
export enum TokenTypes {
22
Text = "Text",
3+
RawContent = "RawContent",
34
OpenTagStart = "OpenTagStart",
45
OpenTagEnd = "OpenTagEnd",
56
CloseTag = "CloseTag",

‎src/index.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
11
export { parse } from "./parser";
22
export { NodeTypes, TokenTypes } from "./constants";
33
export { AnyToken, ParseResult } from "./types";
4+
export { Options } from "./types/parse";
45
export * from "./types/node";

‎src/tokenizer/__tests__/__output__/custom-tag-raw-content.ts‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -110,7 +110,7 @@ const OUTPUT: AnyToken[] = [
110110
},
111111
},
112112
{
113-
type: TokenTypes.Text,
113+
type: TokenTypes.RawContent,
114114
value: `
115115
# Hello, world!
116116

‎src/tokenizer/handlers/custom-tag-raw-content.ts‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -35,11 +35,11 @@ function parseClosingCustomTag(state: TokenizerState) {
3535
if (state.accumulatedContent.value() !== "") {
3636
const position = calculateTokenPosition(state, { keepBuffer: false });
3737
state.tokens.push({
38-
type: TokenTypes.Text,
38+
type: TokenTypes.RawContent,
3939
value: state.accumulatedContent.value(),
4040
range: position.range,
4141
loc: position.loc,
42-
parts: createParts(state, TokenTypes.Text),
42+
parts: createParts(state, TokenTypes.RawContent),
4343
});
4444
}
4545

‎src/tree-constructor/__tests__/__file_snapshots__/construct-tree-Custom-Tag-Raw-Content-0.json‎

Lines changed: 206 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,206 @@
1+
{
2+
"type": "Document",
3+
"range": [
4+
0,
5+
271
6+
],
7+
"children": [
8+
{
9+
"type": "Tag",
10+
"range": [
11+
0,
12+
271
13+
],
14+
"loc": {
15+
"start": {
16+
"line": 1,
17+
"column": 0
18+
},
19+
"end": {
20+
"line": 17,
21+
"column": 11
22+
}
23+
},
24+
"attributes": [
25+
{
26+
"type": "Attribute",
27+
"range": [
28+
10,
29+
18
30+
],
31+
"loc": {
32+
"start": {
33+
"line": 1,
34+
"column": 10
35+
},
36+
"end": {
37+
"line": 1,
38+
"column": 18
39+
}
40+
},
41+
"key": {
42+
"type": "AttributeKey",
43+
"value": "attr",
44+
"loc": {
45+
"start": {
46+
"line": 1,
47+
"column": 10
48+
},
49+
"end": {
50+
"line": 1,
51+
"column": 14
52+
}
53+
},
54+
"range": [
55+
10,
56+
14
57+
],
58+
"parts": []
59+
},
60+
"startWrapper": {
61+
"type": "AttributeValueWrapperStart",
62+
"value": "\"",
63+
"loc": {
64+
"start": {
65+
"line": 1,
66+
"column": 15
67+
},
68+
"end": {
69+
"line": 1,
70+
"column": 16
71+
}
72+
},
73+
"range": [
74+
15,
75+
16
76+
]
77+
},
78+
"value": {
79+
"type": "AttributeValue",
80+
"value": "1",
81+
"loc": {
82+
"start": {
83+
"line": 1,
84+
"column": 16
85+
},
86+
"end": {
87+
"line": 1,
88+
"column": 17
89+
}
90+
},
91+
"range": [
92+
16,
93+
17
94+
],
95+
"parts": []
96+
},
97+
"endWrapper": {
98+
"type": "AttributeValueWrapperEnd",
99+
"value": "\"",
100+
"loc": {
101+
"start": {
102+
"line": 1,
103+
"column": 17
104+
},
105+
"end": {
106+
"line": 1,
107+
"column": 18
108+
}
109+
},
110+
"range": [
111+
17,
112+
18
113+
]
114+
}
115+
}
116+
],
117+
"children": [
118+
{
119+
"type": "RawContent",
120+
"value": "\n# Hello, world!\n\n```cpp{4-6,9}\n#include <iostream>\n\nclass Example {\n Example() {\n std::cout << \"Hello, world!\" << std::endl;\n }\n\n Example(std::string name) {\n std::cout << \"Hello, \" << name << std::endl;\n }\n};\n```\n",
121+
"loc": {
122+
"start": {
123+
"line": 1,
124+
"column": 19
125+
},
126+
"end": {
127+
"line": 17,
128+
"column": 0
129+
}
130+
},
131+
"range": [
132+
19,
133+
260
134+
],
135+
"parts": []
136+
}
137+
],
138+
"openStart": {
139+
"type": "OpenTagStart",
140+
"value": "<markdown",
141+
"loc": {
142+
"start": {
143+
"line": 1,
144+
"column": 0
145+
},
146+
"end": {
147+
"line": 1,
148+
"column": 9
149+
}
150+
},
151+
"range": [
152+
0,
153+
9
154+
]
155+
},
156+
"name": "markdown",
157+
"openEnd": {
158+
"type": "OpenTagEnd",
159+
"value": ">",
160+
"loc": {
161+
"start": {
162+
"line": 1,
163+
"column": 18
164+
},
165+
"end": {
166+
"line": 1,
167+
"column": 19
168+
}
169+
},
170+
"range": [
171+
18,
172+
19
173+
]
174+
},
175+
"selfClosing": false,
176+
"close": {
177+
"type": "CloseTag",
178+
"value": "</markdown>",
179+
"loc": {
180+
"start": {
181+
"line": 17,
182+
"column": 0
183+
},
184+
"end": {
185+
"line": 17,
186+
"column": 11
187+
}
188+
},
189+
"range": [
190+
260,
191+
271
192+
]
193+
}
194+
}
195+
],
196+
"loc": {
197+
"start": {
198+
"line": 1,
199+
"column": 0
200+
},
201+
"end": {
202+
"line": 17,
203+
"column": 11
204+
}
205+
}
206+
}

‎src/tree-constructor/__tests__/construct-tree.spec.ts‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ import TEMPLATE_COMMENT from "../../tokenizer/__tests__/__output__/templates-com
4343
import TEMPLATE_SCRIPT_CONTENT from "../../tokenizer/__tests__/__output__/templates-script-content";
4444
import TEMPLATE_STYLE_CONTENT from "../../tokenizer/__tests__/__output__/templates-style-content";
4545
import TEMPLATE_CONTENT_END from "../../tokenizer/__tests__/__output__/templates-content-end";
46+
import CUSTOM_TAG_RAW_CONTENT from "../../tokenizer/__tests__/__output__/custom-tag-raw-content";
4647
import { clearParent } from "../../utils";
4748
import { toMatchFile } from "jest-file-snapshot";
4849

@@ -78,6 +79,7 @@ describe("construct-tree", () => {
7879
["Templates Script Content", TEMPLATE_SCRIPT_CONTENT],
7980
["Templates Style Content", TEMPLATE_STYLE_CONTENT],
8081
["Templates Content End", TEMPLATE_CONTENT_END],
82+
["Custom Tag Raw Content", CUSTOM_TAG_RAW_CONTENT],
8183
])("%s", (name: string, inputTokens: any) => {
8284
const { ast } = constructTree(inputTokens);
8385
expect(JSON.stringify(clearParent(ast), null, 2)).toMatchFile(undefined, {

‎src/tree-constructor/handlers/tag-content.ts‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -15,6 +15,7 @@ import {
1515
ContextualDoctypeNode,
1616
TextNode,
1717
CompositeToken,
18+
RawContentNode,
1819
} from "../../types";
1920
import {
2021
cloneRange,
@@ -131,6 +132,19 @@ function handleText(
131132
return state;
132133
}
133134

135+
function handleRawContent(
136+
state: ConstructTreeState<ContextualTagNode | ContextualDocumentNode>,
137+
token: CompositeToken<TokenTypes.RawContent>
138+
) {
139+
initChildrenIfNone(state.currentNode);
140+
const rawContentNode = createNodeFrom(token) as RawContentNode;
141+
142+
state.currentNode.children.push(rawContentNode);
143+
state.caretPosition++;
144+
145+
return state;
146+
}
147+
134148
function handleOpenScriptTagStart(
135149
state: ConstructTreeState<ContextualTagNode | ContextualDocumentNode>,
136150
token: Token<TokenTypes.OpenScriptTagStart>
@@ -202,6 +216,10 @@ export function construct(
202216
return handleText(state, token);
203217
}
204218

219+
if (token.type === TokenTypes.RawContent) {
220+
return handleRawContent(state, token);
221+
}
222+
205223
if (token.type === TokenTypes.CloseTag) {
206224
return handleCloseTag(state, token);
207225
}

‎src/types/node.ts‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ export interface DocumentNode extends BaseNode {
1111
type: NodeTypes.Document;
1212
children: Array<
1313
| TextNode
14+
| RawContentNode
1415
| TagNode
1516
| ScriptTagNode
1617
| StyleTagNode
@@ -20,6 +21,7 @@ export interface DocumentNode extends BaseNode {
2021
}
2122

2223
export type TextNode = CompositeNode<NodeTypes.Text>;
24+
export type RawContentNode = CompositeNode<NodeTypes.RawContent>;
2325

2426
export interface TagNode extends BaseNode {
2527
type: NodeTypes.Tag;
@@ -29,7 +31,12 @@ export interface TagNode extends BaseNode {
2931
openEnd: OpenTagEndNode;
3032
close?: CloseTagNode;
3133
children: Array<
32-
TextNode | TagNode | ScriptTagNode | StyleTagNode | CommentNode
34+
| TextNode
35+
| TagNode
36+
| ScriptTagNode
37+
| StyleTagNode
38+
| CommentNode
39+
| RawContentNode
3340
>;
3441
attributes: Array<AttributeNode>;
3542
}

‎src/types/token.ts‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@ export interface CompositeToken<T extends TokenTypes> extends Token<T> {
3030

3131
export type AnyToken =
3232
| CompositeToken<TokenTypes.Text>
33+
| CompositeToken<TokenTypes.RawContent>
3334
| Token<TokenTypes.OpenTagStart>
3435
| Token<TokenTypes.OpenTagEnd>
3536
| Token<TokenTypes.CloseTag>

0 commit comments

Comments
 (0)