Tokenize: remove startpos and endpos from RawToken

The line and column are only necessary for diagnostics so we can defer
the work of computing/tracking these until diagnostics are emitted.

This speeds up Tokenization with RawToken by about 20%

Author: c42f

Tokenize/src/lexer.jl

@@ -221,6 +221,13 @@ function readchar(l::Lexer{I}) where {I <: IO}
     return l.chars[1]
 end
 
+function readchar(l::Lexer{I,RawToken}) where {I <: IO}
+    c = readchar(l.io)
+    l.chars = (l.chars[2], l.chars[3], l.chars[4], c)
+    l.charspos = (l.charspos[2], l.charspos[3], l.charspos[4], position(l.io))
+    return l.chars[1]
+end
+
 readon(l::Lexer{I,RawToken}) where {I <: IO} = l.chars[1]
 function readon(l::Lexer{I,Token}) where {I <: IO}
     if l.charstore.size != 0
@@ -308,9 +315,7 @@ function emit(l::Lexer{IO_t,RawToken}, kind::Kind, err::TokenError = Tokens.NO_E
         end
     end
 
-    tok = RawToken(kind, (l.token_start_row, l.token_start_col),
-                  (l.current_row, l.current_col - 1),
-                  startpos(l), position(l) - 1, err, l.dotop, suffix)
+    tok = RawToken(kind, startpos(l), position(l) - 1, err, l.dotop, suffix)
 
     l.dotop = false
     l.last_token = kind

Tokenize/src/token.jl

@@ -76,19 +76,16 @@ Token() = Token(ERROR, (0,0), (0,0), 0, 0, "", UNKNOWN, false, false)
 struct RawToken <: AbstractToken
     kind::Kind
     # Offsets into a string or buffer
-    startpos::Tuple{Int, Int} # row, col where token starts /end, col is a string index
-    endpos::Tuple{Int, Int}
     startbyte::Int # The byte where the token start in the buffer
     endbyte::Int # The byte where the token ended in the buffer
     token_error::TokenError
     dotop::Bool
     suffix::Bool
 end
-function RawToken(kind::Kind, startposition::Tuple{Int, Int}, endposition::Tuple{Int, Int},
-    startbyte::Int, endbyte::Int)
-RawToken(kind, startposition, endposition, startbyte, endbyte, NO_ERR, false, false)
+function RawToken(kind::Kind, startbyte::Int, endbyte::Int)
+    RawToken(kind, startbyte, endbyte, NO_ERR, false, false)
 end
-RawToken() = RawToken(ERROR, (0,0), (0,0), 0, 0, UNKNOWN, false, false)
+RawToken() = RawToken(ERROR, 0, 0, UNKNOWN, false, false)
 
 
 const _EMPTY_TOKEN = Token()
@@ -177,9 +174,7 @@ end
 Base.print(io::IO, t::Token) = print(io, untokenize(t))
 
 function Base.show(io::IO, t::RawToken)
-    start_r, start_c = startpos(t)
-    end_r, end_c = endpos(t)
-    print(io, rpad(string(start_r, ",", start_c, "-", end_r, ",", end_c), 17, " "))
+    print(io, rpad(string(startbyte(t), "-", endbyte(t)), 11, " "))
     print(io, rpad(kind(t), 15, " "))
 end
 

Tokenize/src/utilities.jl

@@ -198,7 +198,6 @@ eof(io::IO) = Base.eof(io)
 eof(c::Char) = c === EOF_CHAR
 
 readchar(io::IO) = eof(io) ? EOF_CHAR : read(io, Char)
-takechar(io::IO) = (readchar(io); io)
 
 # Checks whether a Char is an operator, which can not be juxtaposed with another
 # Char to be an operator (i.e <=), and can be prefixed by a dot (.)