Merge pull request #18 from c42f/cjf/optimize-token-lookahead

Optimization and cleanup in parse stream

Author: c42f

Tokenize/src/lexer.jl

@@ -524,7 +524,7 @@ function lex_comment(l::Lexer, doemit=true)
         while true
             pc = peekchar(l)
             if pc == '\n' || eof(pc)
-                return doemit ? emit(l, Tokens.COMMENT) : EMPTY_TOKEN(token_type(l))
+                return doemit ? emit(l, Tokens.COMMENT) : EMPTY_TOKEN
             end
             readchar(l)
         end
@@ -534,7 +534,7 @@ function lex_comment(l::Lexer, doemit=true)
         n_start, n_end = 1, 0
         while true
             if eof(c)
-                return doemit ? emit_error(l, Tokens.EOF_MULTICOMMENT) : EMPTY_TOKEN(token_type(l))
+                return doemit ? emit_error(l, Tokens.EOF_MULTICOMMENT) : EMPTY_TOKEN
             end
             nc = readchar(l)
             if c == '#' && nc == '='
@@ -543,7 +543,7 @@ function lex_comment(l::Lexer, doemit=true)
                 n_end += 1
             end
             if n_start == n_end
-                return doemit ? emit(l, Tokens.COMMENT) : EMPTY_TOKEN(token_type(l))
+                return doemit ? emit(l, Tokens.COMMENT) : EMPTY_TOKEN
             end
             pc = c
             c = nc
@@ -852,25 +852,25 @@ function lex_prime(l, doemit = true)
     else
         if accept(l, '\'')
             if accept(l, '\'')
-                return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN(token_type(l))
+                return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN
             else
                 # Empty char literal
                 # Arguably this should be an error here, but we generally
                 # look at the contents of the char literal in the parser,
                 # so we defer erroring until there.
-                return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN(token_type(l))
+                return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN
             end
         end
         while true
             c = readchar(l)
             if eof(c)
-                return doemit ? emit_error(l, Tokens.EOF_CHAR) : EMPTY_TOKEN(token_type(l))
+                return doemit ? emit_error(l, Tokens.EOF_CHAR) : EMPTY_TOKEN
             elseif c == '\\'
                 if eof(readchar(l))
-                    return doemit ? emit_error(l, Tokens.EOF_CHAR) : EMPTY_TOKEN(token_type(l))
+                    return doemit ? emit_error(l, Tokens.EOF_CHAR) : EMPTY_TOKEN
                 end
             elseif c == '\''
-                return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN(token_type(l))
+                return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN
             end
         end
     end

Tokenize/src/token.jl

@@ -67,9 +67,7 @@ function Token(kind::Kind, startbyte::Int, endbyte::Int)
 end
 Token() = Token(ERROR, 0, 0, UNKNOWN, false, false)
 
-
-const _EMPTY_RAWTOKEN = Token()
-EMPTY_TOKEN(::Type{Token}) = _EMPTY_RAWTOKEN
+const EMPTY_TOKEN = Token()
 
 function kind(t::Token)
     isoperator(t.kind) && return OP

Tokenize/src/token_kinds.jl

@@ -1,4 +1,4 @@
-@enum(Kind,
+@enum(Kind::UInt16,
     NONE,      # Placeholder; never emitted by lexer
     ENDMARKER, # EOF
     ERROR,

src/green_tree.jl

@@ -63,10 +63,6 @@ children(node::GreenNode)    = node.args
 span(node::GreenNode)        = node.span
 head(node::GreenNode)        = node.head
 
-# Predicates
-is_trivia(node::GreenNode) = is_trivia(node.head)
-is_error(node::GreenNode)  = is_error(node.head)
-
 Base.summary(node::GreenNode) = summary(node.head)
 
 # Pretty printing