Skip to content

Commit a25e259

Browse files
authored
Merge pull request #18 from c42f/cjf/optimize-token-lookahead
Optimization and cleanup in parse stream
2 parents 77b4044 + f3292d1 commit a25e259

File tree

9 files changed

+267
-181
lines changed

9 files changed

+267
-181
lines changed

Tokenize/src/lexer.jl

Lines changed: 8 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -524,7 +524,7 @@ function lex_comment(l::Lexer, doemit=true)
524524
while true
525525
pc = peekchar(l)
526526
if pc == '\n' || eof(pc)
527-
return doemit ? emit(l, Tokens.COMMENT) : EMPTY_TOKEN(token_type(l))
527+
return doemit ? emit(l, Tokens.COMMENT) : EMPTY_TOKEN
528528
end
529529
readchar(l)
530530
end
@@ -534,7 +534,7 @@ function lex_comment(l::Lexer, doemit=true)
534534
n_start, n_end = 1, 0
535535
while true
536536
if eof(c)
537-
return doemit ? emit_error(l, Tokens.EOF_MULTICOMMENT) : EMPTY_TOKEN(token_type(l))
537+
return doemit ? emit_error(l, Tokens.EOF_MULTICOMMENT) : EMPTY_TOKEN
538538
end
539539
nc = readchar(l)
540540
if c == '#' && nc == '='
@@ -543,7 +543,7 @@ function lex_comment(l::Lexer, doemit=true)
543543
n_end += 1
544544
end
545545
if n_start == n_end
546-
return doemit ? emit(l, Tokens.COMMENT) : EMPTY_TOKEN(token_type(l))
546+
return doemit ? emit(l, Tokens.COMMENT) : EMPTY_TOKEN
547547
end
548548
pc = c
549549
c = nc
@@ -852,25 +852,25 @@ function lex_prime(l, doemit = true)
852852
else
853853
if accept(l, '\'')
854854
if accept(l, '\'')
855-
return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN(token_type(l))
855+
return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN
856856
else
857857
# Empty char literal
858858
# Arguably this should be an error here, but we generally
859859
# look at the contents of the char literal in the parser,
860860
# so we defer erroring until there.
861-
return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN(token_type(l))
861+
return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN
862862
end
863863
end
864864
while true
865865
c = readchar(l)
866866
if eof(c)
867-
return doemit ? emit_error(l, Tokens.EOF_CHAR) : EMPTY_TOKEN(token_type(l))
867+
return doemit ? emit_error(l, Tokens.EOF_CHAR) : EMPTY_TOKEN
868868
elseif c == '\\'
869869
if eof(readchar(l))
870-
return doemit ? emit_error(l, Tokens.EOF_CHAR) : EMPTY_TOKEN(token_type(l))
870+
return doemit ? emit_error(l, Tokens.EOF_CHAR) : EMPTY_TOKEN
871871
end
872872
elseif c == '\''
873-
return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN(token_type(l))
873+
return doemit ? emit(l, Tokens.CHAR) : EMPTY_TOKEN
874874
end
875875
end
876876
end

Tokenize/src/token.jl

Lines changed: 1 addition & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -67,9 +67,7 @@ function Token(kind::Kind, startbyte::Int, endbyte::Int)
6767
end
6868
Token() = Token(ERROR, 0, 0, UNKNOWN, false, false)
6969

70-
71-
const _EMPTY_RAWTOKEN = Token()
72-
EMPTY_TOKEN(::Type{Token}) = _EMPTY_RAWTOKEN
70+
const EMPTY_TOKEN = Token()
7371

7472
function kind(t::Token)
7573
isoperator(t.kind) && return OP

Tokenize/src/token_kinds.jl

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,4 @@
1-
@enum(Kind,
1+
@enum(Kind::UInt16,
22
NONE, # Placeholder; never emitted by lexer
33
ENDMARKER, # EOF
44
ERROR,

src/green_tree.jl

Lines changed: 0 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -63,10 +63,6 @@ children(node::GreenNode) = node.args
6363
span(node::GreenNode) = node.span
6464
head(node::GreenNode) = node.head
6565

66-
# Predicates
67-
is_trivia(node::GreenNode) = is_trivia(node.head)
68-
is_error(node::GreenNode) = is_error(node.head)
69-
7066
Base.summary(node::GreenNode) = summary(node.head)
7167

7268
# Pretty printing

0 commit comments

Comments
 (0)