Avoid crashing on ambiguous tokenization of char vs adjoint (#251)

The tokenizer can't really solve this case, it needs help from the
parser. ie, tighter coupling of tokenizer and parser.

However the workarounds here avoid crashing the parser and even gives
the right result in most cases.

Author: c42f

src/parser.jl

@@ -3389,8 +3389,25 @@ function parse_atom(ps::ParseState, check_identifiers=true)
         # char literal
         bump(ps, TRIVIA_FLAG)
         k = peek(ps)
-        if k == K"Char"
-            bump(ps)
+        if k == K"'"
+            # ''  ==>  (char (error))
+            bump_invisible(ps, K"error", error="empty character literal")
+        elseif k == K"EndMarker"
+            # '   ==>  (char (error))
+            bump_invisible(ps, K"error", error="unterminated character literal")
+        else
+            if k == K"Char"
+                bump(ps)
+            else
+                # FIXME: This case is actually a tokenization error.
+                # Make a best-effort attempt to workaround this for now by
+                # remapping the kind. This needs to be fixed by rewinding the
+                # tokenizer's buffer and re-tokenizing the next token as a
+                # char. (A lot of work for a very obscure edge case)
+                #
+                # x in'c'  ==>  (call-i x in (char 'c'))
+                bump(ps, remap_kind=K"Char")
+            end
             if peek(ps) == K"'"
                 # 'a'         ==>  (char 'a')
                 # 'α'         ==>  (char 'α')
@@ -3401,15 +3418,12 @@ function parse_atom(ps::ParseState, check_identifiers=true)
                 bump_invisible(ps, K"error", TRIVIA_FLAG,
                                error="unterminated character literal")
             end
-        elseif k == K"'"
-            # ''  ==>  (char (error))
-            bump_invisible(ps, K"error", error="empty character literal")
-        else
-            # '   ==>  (char (error))
-            @check k == K"EndMarker"
-            bump_invisible(ps, K"error", error="unterminated character literal")
         end
         emit(ps, mark, K"char")
+    elseif leading_kind == K"Char"
+        # FIXME: This is a tokenization error and should be preceeded with
+        # K"'". However this workaround is better than emitting a bare Char.
+        bump(ps, remap_kind=K"Identifier")
     elseif leading_kind == K":"
         # symbol/expression quote
         # :foo  ==>  (quote-: foo)

test/parser.jl

@@ -948,19 +948,27 @@ tests = [
     end
 end
 
-parseall_test_specs = [
+parsestmt_test_specs = [
     # whitespace before keywords in space-insensitive mode
-    "(y::\nif x z end)" => "(toplevel (parens (::-i y (if x (block z)))))"
+    "(y::\nif x z end)" => "(parens (::-i y (if x (block z))))"
+    # parsing of tricky primes
+    "x in'c'"   => "(call-i x in (char 'c'))"
+    "1where'c'" => "(where 1 (char 'c'))"
+    ":+'y'"     => "(juxtapose (call-post (quote-: +) ') (call-post y '))"
 
     # The following may not be ideal error recovery! But at least the parser
     # shouldn't crash
-    "@(x y)" => "(toplevel (macrocall (parens @x (error-t y))))"
-    "|(&\nfunction" => "(toplevel (call | (& (function (error (error)) (block (error)) (error-t))) (error-t)))"
+    "@(x y)" => "(macrocall (parens @x (error-t y)))"
+    "|(&\nfunction" => "(call | (& (function (error (error)) (block (error)) (error-t))) (error-t))"
+
+    # The following are currently broken but at least the parser shouldn't
+    # crash.
+    "x in' '" => "(call-i x in (char (error))) (error-t ')"
 ]
 
 @testset "Parser does not crash on broken code" begin
-    @testset "$(repr(input))" for (input, output) in parseall_test_specs
-        test_parse(JuliaSyntax.parse_toplevel, input, output)
+    @testset "$(repr(input))" for (input, output) in parsestmt_test_specs
+        test_parse(JuliaSyntax.parse_stmts, input, output)
     end
 end