Remove need for storing newline flag on SyntaxToken

Author: c42f

src/parse_stream.jl

@@ -85,12 +85,11 @@ struct SyntaxToken
     is_dotted::Bool
     is_suffixed::Bool
     had_whitespace::Bool
-    had_newline::Bool
 end
 
-function SyntaxToken(raw::Token, had_whitespace, had_newline)
+function SyntaxToken(raw::Token, had_whitespace)
     SyntaxToken(raw.kind, raw.startbyte + 1, raw.endbyte + 1, raw.dotop, raw.suffix,
-                had_whitespace, had_newline)
+                had_whitespace)
 end
 
 function Base.show(io::IO, tok::SyntaxToken)
@@ -255,15 +254,12 @@ end
 # but this is not a big problem.
 function _buffer_lookahead_tokens(stream::ParseStream)
     had_whitespace = false
-    had_newline    = false
     while true
         raw = Tokenize.Lexers.next_token(stream.lexer)
         k = TzTokens.exactkind(raw)
         was_whitespace = k in (K"Whitespace", K"Comment", K"NewlineWs")
-        was_newline    = k == K"NewlineWs"
         had_whitespace |= was_whitespace
-        had_newline    |= was_newline
-        push!(stream.lookahead, SyntaxToken(raw, had_whitespace, had_newline))
+        push!(stream.lookahead, SyntaxToken(raw, had_whitespace))
         if !was_whitespace
             break
         end

src/parser.jl

@@ -825,6 +825,7 @@ function parse_range(ps::ParseState)
             end
             n_colons += 1
             bump(ps, n_colons == 1 ? EMPTY_FLAGS : TRIVIA_FLAG)
+            had_newline = peek(ps, skip_newlines=false) == K"NewlineWs"
             t = peek_token(ps)
             if is_closing_token(ps, kind(t))
                 # 1: }    ==>  (call-i 1 : (error))
@@ -835,7 +836,7 @@ function parse_range(ps::ParseState)
                 emit_diagnostic(ps, error="found unexpected closing token")
                 return
             end
-            if t.had_newline
+            if had_newline
                 # Error message for people coming from python
                 # 1:\n2 ==> (call-i 1 : (error))
                 emit_diagnostic(ps, whitespace=true,