fix(tokenizer): aggregate shorthand floats, better EOL handling in comments

Author: CodyJasonBennett

src/constants.ts

@@ -99,7 +99,7 @@ export const WGSL_KEYWORDS = [
   'discard',
   'else',
   'enable',
-  'false',
+  // 'false',
   'fn',
   'for',
   'if',
@@ -109,7 +109,7 @@ export const WGSL_KEYWORDS = [
   'return',
   'struct',
   'switch',
-  'true',
+  // 'true',
   'var',
   'while',
 
@@ -501,8 +501,8 @@ export const GLSL_KEYWORDS = [
   'int',
   'void',
   'bool',
-  'true',
-  'false',
+  // 'true',
+  // 'false',
   'invariant',
   'discard',
   'return',

src/tokenizer.ts

@@ -23,6 +23,7 @@ const CR = 13
 const TAB = 9
 const SPACE = 32
 const UNDERSCORE = 95
+const DOT = 46
 const SLASH = 47
 const STAR = 42
 const HASH = 35
@@ -50,7 +51,7 @@ export function tokenize(code: string, index: number = 0): Token[] {
     if (isSpace(char)) {
       while (isSpace(code.charCodeAt(index))) value += code[index++]
       tokens.push({ type: 'whitespace', value })
-    } else if (isDigit(char)) {
+    } else if (isDigit(char) || (char === DOT && isDigit(code.charCodeAt(index)))) {
       while (isFloat(value + code[index]) || isInt(value + code[index])) value += code[index++]
       if (isFloat(value)) tokens.push({ type: 'float', value })
       else tokens.push({ type: 'int', value })
@@ -62,8 +63,8 @@ export function tokenize(code: string, index: number = 0): Token[] {
       else tokens.push({ type: 'identifier', value })
     } else if (char === SLASH && (code.charCodeAt(index) === SLASH || code.charCodeAt(index) === STAR)) {
       const terminator = code.charCodeAt(index) === STAR ? '*/' : '\n'
-      while (!value.endsWith(terminator)) value += code[index++]
-      tokens.push({ type: 'comment', value })
+      while (index < code.length && !value.endsWith(terminator)) value += code[index++]
+      tokens.push({ type: 'comment', value: value.trim() })
     } else {
       for (const symbol of SYMBOLS) {
         if (symbol.length > value.length && code.startsWith(symbol, index - 1)) value = symbol

tests/__snapshots__/index.test.ts.snap

@@ -226,8 +226,7 @@ exports[`tokenize > can tokenize GLSL 1`] = `
   },
   {
     "type": "comment",
-    "value": "// single line
-",
+    "value": "// single line",
   },
   {
     "type": "whitespace",
@@ -330,8 +329,7 @@ exports[`tokenize > can tokenize GLSL 1`] = `
   },
   {
     "type": "comment",
-    "value": "// inline comment
-",
+    "value": "// inline comment",
   },
   {
     "type": "whitespace",
@@ -1703,8 +1701,7 @@ exports[`tokenize > can tokenize WGSL 1`] = `
   },
   {
     "type": "comment",
-    "value": "// single line
-",
+    "value": "// single line",
   },
   {
     "type": "whitespace",

tests/tokenizer.test.ts

@@ -0,0 +1,72 @@
+import { describe, it, expect } from 'vitest'
+import { tokenize, type Token, GLSL_KEYWORDS, GLSL_SYMBOLS, WGSL_KEYWORDS, WGSL_SYMBOLS } from 'shaderkit'
+
+describe('tokenizer', () => {
+  it('can handle whitespace', () => {
+    expect(tokenize(' \n\t')).toStrictEqual<Token[]>([{ type: 'whitespace', value: ' \n\t' }])
+  })
+
+  it('can handle comments', () => {
+    expect(tokenize('// comment')).toStrictEqual<Token[]>([{ type: 'comment', value: '// comment' }])
+    expect(tokenize('/* comment */')).toStrictEqual<Token[]>([{ type: 'comment', value: '/* comment */' }])
+  })
+
+  it('can handle floats', () => {
+    expect(tokenize('0.0')).toStrictEqual<Token[]>([{ type: 'float', value: '0.0' }])
+    expect(tokenize('0.')).toStrictEqual<Token[]>([{ type: 'float', value: '0.' }])
+    expect(tokenize('.0')).toStrictEqual<Token[]>([{ type: 'float', value: '.0' }])
+    expect(tokenize('0.0f')).toStrictEqual<Token[]>([{ type: 'float', value: '0.0f' }])
+    expect(tokenize('0.0F')).toStrictEqual<Token[]>([{ type: 'float', value: '0.0F' }])
+    expect(tokenize('0.0h')).toStrictEqual<Token[]>([{ type: 'float', value: '0.0h' }])
+    expect(tokenize('0.0H')).toStrictEqual<Token[]>([{ type: 'float', value: '0.0H' }])
+    // expect(tokenize('1.23e3')).toStrictEqual<Token[]>([{ type: 'float', value: '1.23e3' }])
+    // expect(tokenize('4e-2')).toStrictEqual<Token[]>([{ type: 'float', value: '4e-2' }])
+    // expect(tokenize('3E+4')).toStrictEqual<Token[]>([{ type: 'float', value: '3E+4' }])
+  })
+
+  it('can handle integers', () => {
+    expect(tokenize('0')).toStrictEqual<Token[]>([{ type: 'int', value: '0' }])
+    expect(tokenize('0u')).toStrictEqual<Token[]>([{ type: 'int', value: '0u' }])
+    expect(tokenize('0U')).toStrictEqual<Token[]>([{ type: 'int', value: '0U' }])
+    expect(tokenize('0i')).toStrictEqual<Token[]>([{ type: 'int', value: '0i' }])
+    expect(tokenize('0I')).toStrictEqual<Token[]>([{ type: 'int', value: '0I' }])
+    // expect(tokenize('0xFF')).toStrictEqual<Token[]>([{ type: 'int', value: '0xFF' }])
+    // expect(tokenize('0XFF')).toStrictEqual<Token[]>([{ type: 'int', value: '0XFF' }])
+  })
+
+  it('can handle identifiers', () => {
+    expect(tokenize('test')).toStrictEqual<Token[]>([{ type: 'identifier', value: 'test' }])
+  })
+
+  it('can handle booleans', () => {
+    expect(tokenize('true')).toStrictEqual<Token[]>([{ type: 'bool', value: 'true' }])
+    expect(tokenize('false')).toStrictEqual<Token[]>([{ type: 'bool', value: 'false' }])
+  })
+
+  it('can handle identifiers', () => {
+    expect(tokenize('test')).toStrictEqual<Token[]>([{ type: 'identifier', value: 'test' }])
+  })
+
+  it('can handle reserved words', () => {
+    // NOTE: language detection is context-sensitive
+    for (const keyword of WGSL_KEYWORDS) {
+      expect(tokenize(`var test: type;${keyword}`).slice(-1)).toStrictEqual<Token[]>([
+        { type: 'keyword', value: keyword.replace('@', '') },
+      ])
+    }
+    for (const keyword of GLSL_KEYWORDS) {
+      expect(tokenize(keyword).slice(-1)).toStrictEqual<Token[]>([{ type: 'keyword', value: keyword.replace('#', '') }])
+    }
+  })
+
+  it('can handle symbols', () => {
+    // NOTE: language detection is context-sensitive
+    const comments = /\/\/|\/\*|\*\//
+    for (const symbol of WGSL_SYMBOLS.filter((s) => !comments.test(s))) {
+      expect(tokenize(`var test: type;${symbol}`).slice(-1)).toStrictEqual<Token[]>([{ type: 'symbol', value: symbol }])
+    }
+    for (const symbol of GLSL_SYMBOLS.filter((s) => !comments.test(s))) {
+      expect(tokenize(symbol).slice(-1)).toStrictEqual<Token[]>([{ type: 'symbol', value: symbol }])
+    }
+  })
+})