Use doc contextual keyword to detect @doc macro in parser

Author: c42f

src/parse_stream.jl

@@ -318,28 +318,6 @@ function peek_token(stream::ParseStream, n::Integer=1; skip_newlines=false)
     stream.lookahead[_lookahead_index(stream, n, skip_newlines)]
 end
 
-function _peek_equal_to(stream, first_byte, len, str)
-    cbuf = codeunits(str)
-    for i = 1:len
-        if stream.textbuf[first_byte + i - 1] != cbuf[i]
-            return false
-        end
-    end
-    return true
-end
-
-"""
-Return true if the node already emitted at `pos` covers the string `str`
-
-This is a hack for edge cases where the parser needs access to interpret normal
-identifiers as contextural keywords. For example, the special parsing rules for
-`@doc` line contination :-(
-"""
-function peek_behind_str(stream::ParseStream, pos::ParseStreamPosition, str::String)
-    s = stream.ranges[pos.output_index]
-    return _peek_equal_to(stream, first_byte(s), span(s), str)
-end
-
 function _peek_behind_fields(ranges, i)
     r = ranges[i]
     return (kind=kind(r),

src/parser.jl

@@ -71,10 +71,6 @@ function peek_token(ps::ParseState, n=1; skip_newlines=nothing)
     peek_token(ps.stream, n, skip_newlines=skip_nl)
 end
 
-function peek_behind_str(ps::ParseState, args...)
-    peek_behind_str(ps.stream, args...)
-end
-
 function peek_behind(ps::ParseState, args...)
     peek_behind(ps.stream, args...)
 end
@@ -1395,7 +1391,7 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
                 # @A.foo a b    ==> (macrocall (. A (quote @foo)) a b)
                 n_args = parse_space_separated_exprs(ps)
                 # TODO: Introduce K"doc" to make this hack less awful.
-                is_doc_macro = peek_behind_str(ps, macro_name_position, "doc")
+                is_doc_macro = peek_behind(ps, macro_name_position).orig_kind == K"doc"
                 if is_doc_macro && n_args == 1
                     # Parse extended @doc args on next line
                     # @doc x\ny      ==>  (macrocall @doc x y)

src/token_kinds.jl

@@ -1,4 +1,7 @@
 # Mapping from token string identifiers to enumeration values as used in @K_str
+#
+# TODO: Unify Tokenize with this approach so we don't need to write these out
+# in two places.
 
 const _str_to_kind = let Ts = TzTokens
 Dict([
@@ -12,8 +15,6 @@ Dict([
 ";"              =>  Ts.SEMICOLON
 
 "BEGIN_KEYWORDS" => Ts.begin_keywords
-"abstract"    =>  Ts.ABSTRACT
-"as"          =>  Ts.AS
 "baremodule"  =>  Ts.BAREMODULE
 "begin"       =>  Ts.BEGIN
 "break"       =>  Ts.BREAK
@@ -35,18 +36,21 @@ Dict([
 "local"       =>  Ts.LOCAL
 "macro"       =>  Ts.MACRO
 "module"      =>  Ts.MODULE
-"mutable"     =>  Ts.MUTABLE
-"new"         =>  Ts.NEW
-"outer"       =>  Ts.OUTER
-"primitive"   =>  Ts.PRIMITIVE
 "quote"       =>  Ts.QUOTE
 "return"      =>  Ts.RETURN
 "struct"      =>  Ts.STRUCT
 "try"         =>  Ts.TRY
-"type"        =>  Ts.TYPE
 "using"       =>  Ts.USING
-"var"         =>  Ts.VAR
 "while"       =>  Ts.WHILE
+# contextural keywords
+"abstract"    =>  Ts.ABSTRACT
+"as"          =>  Ts.AS
+"doc"         =>  Ts.DOC
+"mutable"     =>  Ts.MUTABLE
+"outer"       =>  Ts.OUTER
+"primitive"   =>  Ts.PRIMITIVE
+"type"        =>  Ts.TYPE
+"var"         =>  Ts.VAR
 "END_KEYWORDS" => Ts.end_keywords
 
 # FIXME: Define precisely what Nothing means; integrate better with other tokens.
@@ -875,11 +879,12 @@ const _kind_to_str_unique =
 for kw in split("""
         ( [ { } ] ) @ , ; " \"\"\" ` ```
 
-        as abstract baremodule begin break catch const
+        baremodule begin break catch const
         continue do else elseif end export finally for
         function global if import let local
-        macro module mutable new outer primitive quote
-        return struct try type using var while
+        macro module quote return struct try type using while
+
+        as abstract doc mutable outer primitive type var
 
         block call comparison curly string inert macrocall kw parameters
         toplevel tuple ref vect braces bracescat hcat