@@ -124,7 +124,7 @@ end})
124124
125125local VERSION = " 1.11.1"
126126
127- local MAX_DUPLICATE_FILE_INSERTS = 1000 -- @Incomplete: mak this a parameter for processFile()/processString().
127+ local MAX_DUPLICATE_FILE_INSERTS = 1000 -- @Incomplete: Make this a parameter for processFile()/processString().
128128
129129local KEYWORDS = {
130130 " and" ," break" ," do" ," else" ," elseif" ," end" ," false" ," for" ," function" ," if" ," in" ,
@@ -378,12 +378,12 @@ end
378378
379379local NUM_HEX_FRAC_EXP = (" ^( 0[Xx] ([%dA-Fa-f]*) %.([%dA-Fa-f]+) [Pp]([-+]?[%dA-Fa-f]+) )" ):gsub (" +" , " " )
380380local NUM_HEX_FRAC = (" ^( 0[Xx] ([%dA-Fa-f]*) %.([%dA-Fa-f]+) )" ):gsub (" +" , " " )
381- local NUM_HEX_EXP = (" ^( 0[Xx] ([%dA-Fa-f]+) [Pp]([-+]?[%dA-Fa-f]+) )" ):gsub (" +" , " " )
382- local NUM_HEX = (" ^( 0[Xx] [%dA-Fa-f]+ )" ):gsub (" +" , " " )
381+ local NUM_HEX_EXP = (" ^( 0[Xx] ([%dA-Fa-f]+) %.? [Pp]([-+]?[%dA-Fa-f]+) )" ):gsub (" +" , " " )
382+ local NUM_HEX = (" ^( 0[Xx] [%dA-Fa-f]+ %.? )" ):gsub (" +" , " " )
383383local NUM_DEC_FRAC_EXP = (" ^( %d* %.%d+ [Ee][-+]?%d+ )" ):gsub (" +" , " " )
384384local NUM_DEC_FRAC = (" ^( %d* %.%d+ )" ):gsub (" +" , " " )
385- local NUM_DEC_EXP = (" ^( %d+ [Ee][-+]?%d+ )" ):gsub (" +" , " " )
386- local NUM_DEC = (" ^( %d+ )" ):gsub (" +" , " " )
385+ local NUM_DEC_EXP = (" ^( %d+ %.? [Ee][-+]?%d+ )" ):gsub (" +" , " " )
386+ local NUM_DEC = (" ^( %d+ %.? )" ):gsub (" +" , " " )
387387
388388-- tokens = tokenize( lua, filePath, allowBacktickStrings [, allowPreprocessorTokens=false ] )
389389function tokenize (s , path , allowBacktickStrings , allowMetaTokens )
@@ -395,8 +395,15 @@ function tokenize(s, path, allowBacktickStrings, allowMetaTokens)
395395 local tok
396396 local tokenPos = ptr
397397
398+ -- Whitespace.
399+ if s :find (" ^%s" , ptr ) then
400+ local i1 , i2 , whitespace = s :find (" ^(%s+)" , ptr )
401+
402+ ptr = i2 + 1
403+ tok = {type = " whitespace" , representation = whitespace , value = whitespace }
404+
398405 -- Identifier/keyword.
399- if s :find (" ^[%a_]" , ptr ) then
406+ elseif s :find (" ^[%a_]" , ptr ) then
400407 local i1 , i2 , word = s :find (" ^([%a_][%w_]*)" , ptr )
401408 ptr = i2 + 1
402409
@@ -584,13 +591,6 @@ function tokenize(s, path, allowBacktickStrings, allowMetaTokens)
584591 ptr = i2 + 1
585592 tok = {type = " string" , representation = repr , value = v , long = false }
586593
587- -- Whitespace.
588- elseif s :find (" ^%s" , ptr ) then
589- local i1 , i2 , whitespace = s :find (" ^(%s+)" , ptr )
590-
591- ptr = i2 + 1
592- tok = {type = " whitespace" , representation = whitespace , value = whitespace }
593-
594594 -- Punctuation etc.
595595 elseif s :find (" ^%.%.%." , ptr ) then
596596 local repr = s :sub (ptr , ptr + 2 )
@@ -2178,7 +2178,7 @@ return lib
21782178
21792179--[[ !===========================================================
21802180
2181- Copyright © 2018-2019 Marcus 'ReFreezed' Thunström
2181+ Copyright © 2018-2020 Marcus 'ReFreezed' Thunström
21822182
21832183Permission is hereby granted, free of charge, to any person obtaining a copy
21842184of this software and associated documentation files (the "Software"), to deal
0 commit comments