Fix handle CRLF (#15)

Author: yeonjuan

src/tokenizer/__tests__/__input__/attributes-multiline.html

@@ -0,0 +1,6 @@
+<div
+  id="foo"
+  style="bar"
+  bool
+>
+</div>

src/tokenizer/__tests__/__output__/attributes-multiline-crlf.ts

@@ -0,0 +1,244 @@
+import { TokenTypes } from "../../../constants";
+
+export default [
+  {
+    type: TokenTypes.OpenTagStart,
+    range: [0, 4],
+    loc: {
+      start: {
+        column: 0,
+        line: 1,
+      },
+      end: {
+        column: 4,
+        line: 1,
+      },
+    },
+    value: "<div",
+  },
+  {
+    type: TokenTypes.AttributeKey,
+    range: [8, 10],
+    value: "id",
+    loc: {
+      start: {
+        line: 2,
+        column: 2,
+      },
+      end: {
+        line: 2,
+        column: 4,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeAssignment,
+    range: [10, 11],
+    value: "=",
+    loc: {
+      start: {
+        line: 2,
+        column: 4,
+      },
+      end: {
+        line: 2,
+        column: 5,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeValueWrapperStart,
+    value: '"',
+    range: [11, 12],
+    loc: {
+      start: {
+        line: 2,
+        column: 5,
+      },
+      end: {
+        line: 2,
+        column: 6,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeValue,
+    value: "foo",
+    range: [12, 15],
+    loc: {
+      start: {
+        line: 2,
+        column: 6,
+      },
+      end: {
+        line: 2,
+        column: 9,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeValueWrapperEnd,
+    value: '"',
+    range: [15, 16],
+    loc: {
+      start: {
+        line: 2,
+        column: 9,
+      },
+      end: {
+        line: 2,
+        column: 10,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeKey,
+    value: "style",
+    range: [20, 25],
+    loc: {
+      start: {
+        line: 3,
+        column: 2,
+      },
+      end: {
+        line: 3,
+        column: 7,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeAssignment,
+    value: "=",
+    range: [25, 26],
+    loc: {
+      start: {
+        line: 3,
+        column: 7,
+      },
+      end: {
+        line: 3,
+        column: 8,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeValueWrapperStart,
+    value: '"',
+    range: [26, 27],
+    loc: {
+      start: {
+        line: 3,
+        column: 8,
+      },
+      end: {
+        line: 3,
+        column: 9,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeValue,
+    value: "bar",
+    range: [27, 30],
+    loc: {
+      start: {
+        line: 3,
+        column: 9,
+      },
+      end: {
+        line: 3,
+        column: 12,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeValueWrapperEnd,
+    value: '"',
+    range: [30, 31],
+    loc: {
+      start: {
+        line: 3,
+        column: 12,
+      },
+      end: {
+        line: 3,
+        column: 13,
+      },
+    },
+  },
+  {
+    type: TokenTypes.AttributeKey,
+    value: "bool",
+    range: [35, 39],
+    loc: {
+      start: {
+        line: 4,
+        column: 2,
+      },
+      end: {
+        line: 4,
+        column: 6,
+      },
+    },
+  },
+  {
+    type: TokenTypes.OpenTagEnd,
+    value: ">",
+    range: [41, 42],
+    loc: {
+      start: {
+        line: 5,
+        column: 0,
+      },
+      end: {
+        line: 5,
+        column: 1,
+      },
+    },
+  },
+  {
+    type: TokenTypes.Text,
+    value: "\r\n",
+    range: [42, 44],
+    loc: {
+      start: {
+        line: 5,
+        column: 1,
+      },
+      end: {
+        line: 6,
+        column: 0,
+      },
+    },
+  },
+  {
+    type: TokenTypes.CloseTag,
+    value: "</div>",
+    range: [44, 50],
+    loc: {
+      start: {
+        line: 6,
+        column: 0,
+      },
+      end: {
+        line: 6,
+        column: 6,
+      },
+    },
+  },
+  {
+    type: TokenTypes.Text,
+    value: "\r\n",
+    range: [50, 52],
+    loc: {
+      start: {
+        line: 6,
+        column: 6,
+      },
+      end: {
+        line: 7,
+        column: 0,
+      },
+    },
+  },
+];

src/tokenizer/__tests__/tokenize.spec.ts

@@ -18,6 +18,7 @@ import TAGS_REGISTER from "./__output__/tags-register";
 import VOID_TAGS from "./__output__/void-tags";
 import EMPTY from "./__output__/empty";
 import SVG from "./__output__/svg";
+import ATTRIBUTES_MULTILINE_CRLF from "./__output__/attributes-multiline-crlf";
 
 describe("tokenize", () => {
   test.each(
@@ -107,12 +108,31 @@ describe("tokenize", () => {
         "Svg",
         "svg.html",
         SVG,
+      ],
+      [
+        "Attributes multiline CRLF",
+        "attributes-multiline.html",
+        ATTRIBUTES_MULTILINE_CRLF,
+        (html: string) => {
+          return html.replace(/\n/gi, "\r\n");
+        }
       ]
     ]
-  )("%s", (name, input, output) => {
-    const inputPath = path.join(__dirname, "__input__", input);
-    const html = fs.readFileSync(inputPath, "utf-8");
-    const { tokens } = tokenize(html, undefined);
-    expect(tokens).toEqual(output);
-  });
+  )(
+    "%s",
+    (
+      name,
+      input,
+      output,
+      process: null | ((html: string) => string) = null
+    ) => {
+      const inputPath = path.join(__dirname, "__input__", input);
+      let html = fs.readFileSync(inputPath, "utf-8");
+      if (process) {
+        html = process(html);
+      }
+      const { tokens } = tokenize(html, undefined);
+      expect(tokens).toEqual(output);
+    }
+  );
 });

src/tokenizer/handlers/attribute-key.ts

@@ -1,6 +1,6 @@
 import { TokenizerContextTypes } from "../../constants";
 import { TokenTypes } from "../../constants/token-types";
-import { calculateTokenPosition } from "../../utils";
+import { calculateTokenPosition, isWhitespace } from "../../utils";
 import { AnyToken, TokenizerState } from "../../types";
 
 export function parse(
@@ -18,14 +18,7 @@ export function parse(
 }
 
 function isKeyBreak(chars: string): boolean {
-  return (
-    chars === "=" ||
-    chars === " " ||
-    chars === "\n" ||
-    chars === "\t" ||
-    chars === "/" ||
-    chars === ">"
-  );
+  return chars === "=" || chars === "/" || chars === ">" || isWhitespace(chars);
 }
 
 function parseKeyEnd(state: TokenizerState, tokens: AnyToken[]) {

src/tokenizer/handlers/open-tag-start.ts

@@ -20,6 +20,7 @@ export function parse(
   if (chars === ">" || chars === "/") {
     return parseTagEnd(state, tokens);
   }
+
   if (isWhitespace(chars)) {
     return parseWhitespace(state, tokens);
   }