@@ -38,7 +38,7 @@ export class Lexer {
3838 lineStart : number ;
3939
4040 constructor ( source : Source ) {
41- const startOfFileToken = new Token ( TokenKind . SOF , 0 , 0 , 0 , 0 ) ;
41+ const startOfFileToken = new Token ( TokenKind . SOF , 0 , 0 , 0 , 0 , undefined ) ;
4242
4343 this . source = source ;
4444 this . lastToken = startOfFileToken ;
@@ -174,11 +174,10 @@ function createToken(
174174 kind : TokenKind ,
175175 start : number ,
176176 end : number ,
177- value ? : string ,
177+ value : string | undefined ,
178178) : Token {
179- const line = lexer . line ;
180- const col = 1 + start - lexer . lineStart ;
181- return new Token ( kind , start , end , line , col , value ) ;
179+ const col = start - ( lexer . lineStart - 1 ) ;
180+ return new Token ( kind , start , end , lexer . line , col , value ) ;
182181}
183182
184183/**
@@ -248,39 +247,123 @@ function readNextToken(lexer: Lexer, start: number): Token {
248247 //
249248 // Punctuator :: one of ! $ & ( ) ... : = @ [ ] { | }
250249 case 0x0021 : // !
251- return createToken ( lexer , TokenKind . BANG , position , position + 1 ) ;
250+ return createToken (
251+ lexer ,
252+ TokenKind . BANG ,
253+ position ,
254+ position + 1 ,
255+ undefined ,
256+ ) ;
252257 case 0x0024 : // $
253- return createToken ( lexer , TokenKind . DOLLAR , position , position + 1 ) ;
258+ return createToken (
259+ lexer ,
260+ TokenKind . DOLLAR ,
261+ position ,
262+ position + 1 ,
263+ undefined ,
264+ ) ;
254265 case 0x0026 : // &
255- return createToken ( lexer , TokenKind . AMP , position , position + 1 ) ;
266+ return createToken (
267+ lexer ,
268+ TokenKind . AMP ,
269+ position ,
270+ position + 1 ,
271+ undefined ,
272+ ) ;
256273 case 0x0028 : // (
257- return createToken ( lexer , TokenKind . PAREN_L , position , position + 1 ) ;
274+ return createToken (
275+ lexer ,
276+ TokenKind . PAREN_L ,
277+ position ,
278+ position + 1 ,
279+ undefined ,
280+ ) ;
258281 case 0x0029 : // )
259- return createToken ( lexer , TokenKind . PAREN_R , position , position + 1 ) ;
282+ return createToken (
283+ lexer ,
284+ TokenKind . PAREN_R ,
285+ position ,
286+ position + 1 ,
287+ undefined ,
288+ ) ;
260289 case 0x002e : // .
261290 if (
262291 body . charCodeAt ( position + 1 ) === 0x002e &&
263292 body . charCodeAt ( position + 2 ) === 0x002e
264293 ) {
265- return createToken ( lexer , TokenKind . SPREAD , position , position + 3 ) ;
294+ return createToken (
295+ lexer ,
296+ TokenKind . SPREAD ,
297+ position ,
298+ position + 3 ,
299+ undefined ,
300+ ) ;
266301 }
267302 break ;
268303 case 0x003a : // :
269- return createToken ( lexer , TokenKind . COLON , position , position + 1 ) ;
304+ return createToken (
305+ lexer ,
306+ TokenKind . COLON ,
307+ position ,
308+ position + 1 ,
309+ undefined ,
310+ ) ;
270311 case 0x003d : // =
271- return createToken ( lexer , TokenKind . EQUALS , position , position + 1 ) ;
312+ return createToken (
313+ lexer ,
314+ TokenKind . EQUALS ,
315+ position ,
316+ position + 1 ,
317+ undefined ,
318+ ) ;
272319 case 0x0040 : // @
273- return createToken ( lexer , TokenKind . AT , position , position + 1 ) ;
320+ return createToken (
321+ lexer ,
322+ TokenKind . AT ,
323+ position ,
324+ position + 1 ,
325+ undefined ,
326+ ) ;
274327 case 0x005b : // [
275- return createToken ( lexer , TokenKind . BRACKET_L , position , position + 1 ) ;
328+ return createToken (
329+ lexer ,
330+ TokenKind . BRACKET_L ,
331+ position ,
332+ position + 1 ,
333+ undefined ,
334+ ) ;
276335 case 0x005d : // ]
277- return createToken ( lexer , TokenKind . BRACKET_R , position , position + 1 ) ;
336+ return createToken (
337+ lexer ,
338+ TokenKind . BRACKET_R ,
339+ position ,
340+ position + 1 ,
341+ undefined ,
342+ ) ;
278343 case 0x007b : // {
279- return createToken ( lexer , TokenKind . BRACE_L , position , position + 1 ) ;
344+ return createToken (
345+ lexer ,
346+ TokenKind . BRACE_L ,
347+ position ,
348+ position + 1 ,
349+ undefined ,
350+ ) ;
280351 case 0x007c : // |
281- return createToken ( lexer , TokenKind . PIPE , position , position + 1 ) ;
352+ return createToken (
353+ lexer ,
354+ TokenKind . PIPE ,
355+ position ,
356+ position + 1 ,
357+ undefined ,
358+ ) ;
282359 case 0x007d : // }
283- return createToken ( lexer , TokenKind . BRACE_R , position , position + 1 ) ;
360+ return createToken (
361+ lexer ,
362+ TokenKind . BRACE_R ,
363+ position ,
364+ position + 1 ,
365+ undefined ,
366+ ) ;
284367 // StringValue
285368 case 0x0022 : // "
286369 if (
@@ -313,7 +396,7 @@ function readNextToken(lexer: Lexer, start: number): Token {
313396 ) ;
314397 }
315398
316- return createToken ( lexer , TokenKind . EOF , bodyLength , bodyLength ) ;
399+ return createToken ( lexer , TokenKind . EOF , bodyLength , bodyLength , undefined ) ;
317400}
318401
319402/**
@@ -353,7 +436,7 @@ function readComment(lexer: Lexer, start: number): Token {
353436 TokenKind . COMMENT ,
354437 start ,
355438 position ,
356- body . slice ( start + 1 , position ) ,
439+ body . substring ( start + 1 , position ) ,
357440 ) ;
358441}
359442
@@ -454,7 +537,7 @@ function readNumber(lexer: Lexer, start: number, firstCode: number): Token {
454537 isFloat ? TokenKind . FLOAT : TokenKind . INT ,
455538 start ,
456539 position ,
457- body . slice ( start , position ) ,
540+ body . substring ( start , position ) ,
458541 ) ;
459542}
460543
@@ -515,13 +598,13 @@ function readString(lexer: Lexer, start: number): Token {
515598
516599 // Closing Quote (")
517600 if ( code === 0x0022 ) {
518- value += body . slice ( chunkStart , position ) ;
601+ value += body . substring ( chunkStart , position ) ;
519602 return createToken ( lexer , TokenKind . STRING , start , position + 1 , value ) ;
520603 }
521604
522605 // Escape Sequence (\)
523606 if ( code === 0x005c ) {
524- value += body . slice ( chunkStart , position ) ;
607+ value += body . substring ( chunkStart , position ) ;
525608 const escape =
526609 body . charCodeAt ( position + 1 ) === 0x0075 // u
527610 ? body . charCodeAt ( position + 2 ) === 0x007b // {
@@ -593,7 +676,7 @@ function readEscapedUnicodeVariableWidth(
593676 throw syntaxError (
594677 lexer . source ,
595678 position ,
596- `Invalid Unicode escape sequence: "${ body . slice (
679+ `Invalid Unicode escape sequence: "${ body . substring (
597680 position ,
598681 position + size ,
599682 ) } ".`,
@@ -635,7 +718,10 @@ function readEscapedUnicodeFixedWidth(
635718 throw syntaxError (
636719 lexer . source ,
637720 position ,
638- `Invalid Unicode escape sequence: "${ body . slice ( position , position + 6 ) } ".` ,
721+ `Invalid Unicode escape sequence: "${ body . substring (
722+ position ,
723+ position + 6 ,
724+ ) } ".`,
639725 ) ;
640726}
641727
@@ -717,7 +803,7 @@ function readEscapedCharacter(lexer: Lexer, position: number): EscapeSequence {
717803 throw syntaxError (
718804 lexer . source ,
719805 position ,
720- `Invalid character escape sequence: "${ body . slice (
806+ `Invalid character escape sequence: "${ body . substring (
721807 position ,
722808 position + 2 ,
723809 ) } ".`,
@@ -755,7 +841,7 @@ function readBlockString(lexer: Lexer, start: number): Token {
755841 body . charCodeAt ( position + 1 ) === 0x0022 &&
756842 body . charCodeAt ( position + 2 ) === 0x0022
757843 ) {
758- currentLine += body . slice ( chunkStart , position ) ;
844+ currentLine += body . substring ( chunkStart , position ) ;
759845 blockLines . push ( currentLine ) ;
760846
761847 const token = createToken (
@@ -779,15 +865,15 @@ function readBlockString(lexer: Lexer, start: number): Token {
779865 body . charCodeAt ( position + 2 ) === 0x0022 &&
780866 body . charCodeAt ( position + 3 ) === 0x0022
781867 ) {
782- currentLine += body . slice ( chunkStart , position ) ;
868+ currentLine += body . substring ( chunkStart , position ) ;
783869 chunkStart = position + 1 ; // skip only slash
784870 position += 4 ;
785871 continue ;
786872 }
787873
788874 // LineTerminator
789875 if ( code === 0x000a || code === 0x000d ) {
790- currentLine += body . slice ( chunkStart , position ) ;
876+ currentLine += body . substring ( chunkStart , position ) ;
791877 blockLines . push ( currentLine ) ;
792878
793879 if ( code === 0x000d && body . charCodeAt ( position + 1 ) === 0x000a ) {
@@ -849,6 +935,6 @@ function readName(lexer: Lexer, start: number): Token {
849935 TokenKind . NAME ,
850936 start ,
851937 position ,
852- body . slice ( start , position ) ,
938+ body . substring ( start , position ) ,
853939 ) ;
854940}
0 commit comments