Bugfix: Always emit a token from parse_atom (#47)

If a closing token is found in parse_atom, we still need to emit a
nontrivia token to make the resulting parse tree consistent. We choose
an invisible token for this.

Also add various minor fixes to error recovory and a few additional
tests in parse_atom and elsewhere.

Author: c42f

src/parser.jl

@@ -151,7 +151,7 @@ function bump_closing_token(ps, closing_kind)
     end
     # mark as trivia => ignore in AST.
     emit(ps, mark, K"error", TRIVIA_FLAG,
-         error="Expected `$(untokenize(closing_kind))` but got unexpected tokens")
+         error="Expected `$(untokenize(closing_kind))`")
     if peek(ps) == closing_kind
         bump(ps, TRIVIA_FLAG)
     end
@@ -2798,11 +2798,11 @@ function parse_cat(ps::ParseState, closer, end_is_symbol)
     if k == K"," || (is_closing_token(ps, k) && k != K";")
         if k == K","
             # [x,]  ==>  (vect x)
-            # [x    ==>  (vect x)
             bump(ps, TRIVIA_FLAG)
         end
         # [x]      ==>  (vect x)
         # [x \n ]  ==>  (vect x)
+        # [x       ==>  (vect x (error-t))
         parse_vect(ps, closer)
     elseif k == K"for"
         # [x for a in as]  ==>  (comprehension (generator x (= a as)))
@@ -2957,6 +2957,9 @@ function parse_brackets(after_parse::Function,
             num_semis += 1
             bump(ps, TRIVIA_FLAG)
             bump_trivia(ps)
+        elseif is_closing_token(ps, k)
+            # Error; handled below in bump_closing_token
+            break
         else
             mark = position(ps)
             eq_pos = parse_eq_star(ps)
@@ -3279,8 +3282,11 @@ function parse_atom(ps::ParseState, check_identifiers=true)
         end
         emit(ps, mark, K"quote")
     elseif leading_kind == K"=" && is_plain_equals(peek_token(ps))
-        bump(ps, TRIVIA_FLAG, error="unexpected `=`")
+        # =   ==> (error =)
+        bump(ps, error="unexpected `=`")
     elseif leading_kind == K"Identifier"
+        # xx  ==>  xx
+        # x₁  ==>  x₁
         bump(ps)
     elseif is_operator(leading_kind)
         if check_identifiers && is_syntactic_operator(leading_kind)
@@ -3375,14 +3381,16 @@ function parse_atom(ps::ParseState, check_identifiers=true)
         parse_string(ps, true)
         emit(ps, mark, K"macrocall")
     elseif is_literal(leading_kind)
+        # 42   ==>  42
         bump(ps)
     elseif is_closing_token(ps, leading_kind)
         # Leave closing token in place for other productions to 
-        # recover with (??)
+        # recover with
+        # )  ==>  error
         msg = leading_kind == K"EndMarker" ?
               "premature end of input" :
               "unexpected closing token"
-        emit_diagnostic(ps, error=msg)
+        bump_invisible(ps, K"error", error=msg)
     else
         bump(ps, error="invalid syntax atom")
     end

test/parser.jl

@@ -458,6 +458,8 @@ tests = [
         "function f() where T   end"  =>  "(function (where (call f) T) (block))"
         "function f() \n a \n b end"  =>  "(function (call f) (block a b))"
         "function f() end"       =>  "(function (call f) (block))"
+        # Errors
+        "function"            => "(function (error (error)) (block (error)) (error-t))"
     ],
     JuliaSyntax.parse_try => [
         "try \n x \n catch e \n y \n finally \n z end" =>
@@ -560,8 +562,13 @@ tests = [
         ":foo"   => "(quote foo)"
         ": foo"  => "(quote (error-t) foo)"
         # Literal colons
-        ":)"   => ":"
-        ": end"   => ":"
+        ":)"     => ":"
+        ": end"  => ":"
+        # plain equals
+        "="      => "(error =)"
+        # Identifiers
+        "xx"     => "xx"
+        "x₁"     => "x₁"
         # var syntax
         """var"x" """  =>  "x"
         """var"x"+"""  =>  "x"
@@ -586,14 +593,16 @@ tests = [
         ":.="  =>  "(quote .=)"
         # Special symbols quoted
         ":end" => "(quote end)"
-        ":(end)" => "(quote (error (end)))"
+        ":(end)" => "(quote (error-t))"
         ":<:"  => "(quote <:)"
+        # unexpect =
+        "="    => "(error =)"
         # parse_cat
         "[]"        =>  "(vect)"
         "[x,]"      =>  "(vect x)"
         "[x]"       =>  "(vect x)"
-        "[x"        =>  "(vect x (error-t))"
         "[x \n ]"   =>  "(vect x)"
+        "[x"        =>  "(vect x (error-t))"
         "[x \n\n ]" =>  "(vect x)"
         "[x for a in as]"  =>  "(comprehension (generator x (= a as)))"
         "[x \n\n for a in as]"  =>  "(comprehension (generator x (= a as)))"
@@ -625,8 +634,10 @@ tests = [
         "``"         =>  "(macrocall :(Core.var\"@cmd\") \"\")"
         "`cmd`"      =>  "(macrocall :(Core.var\"@cmd\") \"cmd\")"
         "```cmd```"  =>  "(macrocall :(Core.var\"@cmd\") \"cmd\")"
-        # Errors
-        ": foo" => "(quote (error-t) foo)"
+        # literals
+        "42"   => "42"
+        # closing tokens
+        ")"    => "(error)"
     ],
     JuliaSyntax.parse_atom => [
         # Actually parse_array

test/test_utils.jl

@@ -243,7 +243,7 @@ function itest_parse(production, code; version::VersionNumber=v"1.6")
         show(stdout, MIME"text/plain"(), f_ex)
 
         printstyled(stdout, "\n\n# Diff of AST dump:\n", color=:red)
-        show_expr_text_diff(showfunc, ex, f_ex, context=10)
+        show_expr_text_diff(show, ex, f_ex, context=10)
         # return (ex, f_ex)
         # return (code, stream, t, s, ex)
     end