Treat UTF-8 BOM as whitespace (#26)

Co-authored-by: Sebastian Pfitzner <[email protected]>

Author: c42f

Tokenize/src/lexer.jl

@@ -16,7 +16,7 @@ export tokenize
 @inline ishex(c::Char) = isdigit(c) || ('a' <= c <= 'f') || ('A' <= c <= 'F')
 @inline isbinary(c::Char) = c == '0' || c == '1'
 @inline isoctal(c::Char) =  '0' ≤ c ≤ '7'
-@inline iswhitespace(c::Char) = Base.isspace(c)
+@inline iswhitespace(c::Char) = Base.isspace(c) || c === '\ufeff'
 
 struct StringState
     triplestr::Bool

Tokenize/test/lexer.jl

@@ -868,3 +868,16 @@ end
         check_kw_hashes(String([cs...]) for cs in Iterators.product(['a':'z' for _ in 1:len]...))
     end
 end
+
+
+@testset "UTF-8 BOM" begin
+    @test Tokenize.Tokens.kind.(collect(tokenize("\ufeff[1\ufeff2]"))) == [
+        Tokens.WHITESPACE,
+        Tokens.LSQUARE,
+        Tokens.INTEGER,
+        Tokens.WHITESPACE,
+        Tokens.INTEGER,
+        Tokens.RSQUARE,
+        Tokens.ENDMARKER
+    ]
+end