fix(tokenizer): greedy match for complex numbers

Author: CodyJasonBennett

src/tokenizer.ts

@@ -10,8 +10,8 @@ export interface Token<T = TokenType, V = string> {
 // Checks for WGSL-specific `fn foo(`, `var bar =`, `let baz =`, `const qux =`
 const WGSL_REGEX = /\bfn\s+\w+\s*\(|\b(var|let|const)\s+\w+\s*[:=]/
 
-const FLOAT_REGEX = /^(\d+\.\d*|\d*\.\d+)([eEpP][-+]?\d+)?[fFhH]?$/
-const INT_REGEX = /^(0[xX][\w\d]+|\d+)[iIuU]?$/
+const FLOAT_REGEX = /((\d+\.\d*|\d*\.\d+)([eEpP][-+]?\d+)?|\d+[eEpP][-+]?\d+)[fFhH]?/y
+const INT_REGEX = /(0[xX][\w\d]+|\d+)[iIuU]?/y
 const BOOL_REGEX = /^(true|false)$/
 
 const ZERO = 48
@@ -36,6 +36,12 @@ const isSpace = (c: number) => isLine(c) || c === TAB || c === SPACE
 const isIdent = (c: number) => isAlpha(c) || isDigit(c) || c === UNDERSCORE
 const isMacro = (c: number) => c === HASH || c === AT
 
+// https://mrale.ph/blog/2016/11/23/making-less-dart-faster.html
+function matchAsPrefix(regex: RegExp, string: string, start: number): string | undefined {
+  regex.lastIndex = start
+  return regex.exec(string)?.[0]
+}
+
 /**
  * Tokenizes a string of GLSL or WGSL code.
  */
@@ -52,9 +58,13 @@ export function tokenize(code: string, index: number = 0): Token[] {
       while (isSpace(code.charCodeAt(index))) value += code[index++]
       tokens.push({ type: 'whitespace', value })
     } else if (isDigit(char) || (char === DOT && isDigit(code.charCodeAt(index)))) {
-      while (FLOAT_REGEX.test(value + code[index]) || INT_REGEX.test(value + code[index])) value += code[index++]
-      if (FLOAT_REGEX.test(value)) tokens.push({ type: 'float', value })
-      else tokens.push({ type: 'int', value })
+      if ((value = matchAsPrefix(FLOAT_REGEX, code, index - 1)!)) {
+        index = FLOAT_REGEX.lastIndex
+        tokens.push({ type: 'float', value })
+      } else if ((value = matchAsPrefix(INT_REGEX, code, index - 1)!)) {
+        index = INT_REGEX.lastIndex
+        tokens.push({ type: 'int', value })
+      }
     } else if (isIdent(char)) {
       while (isIdent(code.charCodeAt(index))) value += code[index++]
       if (BOOL_REGEX.test(value)) tokens.push({ type: 'bool', value })

tests/tokenizer.test.ts

@@ -19,9 +19,9 @@ describe('tokenizer', () => {
     expect(tokenize('0.0F')).toStrictEqual<Token[]>([{ type: 'float', value: '0.0F' }])
     expect(tokenize('0.0h')).toStrictEqual<Token[]>([{ type: 'float', value: '0.0h' }])
     expect(tokenize('0.0H')).toStrictEqual<Token[]>([{ type: 'float', value: '0.0H' }])
-    // expect(tokenize('1.23e3')).toStrictEqual<Token[]>([{ type: 'float', value: '1.23e3' }])
-    // expect(tokenize('4e-2')).toStrictEqual<Token[]>([{ type: 'float', value: '4e-2' }])
-    // expect(tokenize('3E+4')).toStrictEqual<Token[]>([{ type: 'float', value: '3E+4' }])
+    expect(tokenize('1.23e3')).toStrictEqual<Token[]>([{ type: 'float', value: '1.23e3' }])
+    expect(tokenize('4e-2')).toStrictEqual<Token[]>([{ type: 'float', value: '4e-2' }])
+    expect(tokenize('3E+4')).toStrictEqual<Token[]>([{ type: 'float', value: '3E+4' }])
   })
 
   it('can handle integers', () => {
@@ -30,8 +30,8 @@ describe('tokenizer', () => {
     expect(tokenize('0U')).toStrictEqual<Token[]>([{ type: 'int', value: '0U' }])
     expect(tokenize('0i')).toStrictEqual<Token[]>([{ type: 'int', value: '0i' }])
     expect(tokenize('0I')).toStrictEqual<Token[]>([{ type: 'int', value: '0I' }])
-    // expect(tokenize('0xFF')).toStrictEqual<Token[]>([{ type: 'int', value: '0xFF' }])
-    // expect(tokenize('0XFF')).toStrictEqual<Token[]>([{ type: 'int', value: '0XFF' }])
+    expect(tokenize('0xFF')).toStrictEqual<Token[]>([{ type: 'int', value: '0xFF' }])
+    expect(tokenize('0XFF')).toStrictEqual<Token[]>([{ type: 'int', value: '0XFF' }])
   })
 
   it('can handle identifiers', () => {