Parse docstring attachment as K"doc" kind (#217)

With this change `"str" f` now parses as

    (doc (string "str") f)

This better represents the surface syntax which isn't an explicit macro
call but a juxtaposition at top level.

Author: c42f

src/expr.jl

@@ -44,8 +44,6 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
                       val isa UInt128 ? Symbol("@uint128_str") :
                       Symbol("@big_str")
             return Expr(:macrocall, GlobalRef(Core, macname), nothing, str)
-        elseif kind(node) == K"core_@doc"
-            return GlobalRef(Core, Symbol("@doc"))
         elseif kind(node) == K"core_@cmd"
             return GlobalRef(Core, Symbol("@cmd"))
         elseif kind(node) == K"MacroName" && val === Symbol("@.")
@@ -156,6 +154,9 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
     if headsym === :macrocall
         reorder_parameters!(args, 2)
         insert!(args, 2, loc)
+    elseif headsym === :doc
+        return Expr(:macrocall, GlobalRef(Core, Symbol("@doc")),
+                    loc, args...)
     elseif headsym in (:dotcall, :call)
         # Julia's standard `Expr` ASTs have children stored in a canonical
         # order which is often not always source order. We permute the children

src/kinds.jl

@@ -864,7 +864,6 @@ const _kind_names =
             "MacroName"
             "StringMacroName"
             "CmdMacroName"
-            "core_@doc"
             "core_@cmd"
             "core_@int128_str"
             "core_@uint128_str"

src/parser.jl

@@ -477,7 +477,7 @@ end
 #
 # a;b;c   ==>  (toplevel a b c)
 # a;;;b;; ==>  (toplevel a b)
-# "x" a ; "y" b ==>  (toplevel (macrocall core_@doc "x" a) (macrocall core_@doc "y" b))
+# "x" a ; "y" b ==>  (toplevel (doc (string "x") a) (doc (string "y") b))
 #
 # flisp: parse-stmts
 function parse_stmts(ps::ParseState)
@@ -500,12 +500,10 @@ function parse_stmts(ps::ParseState)
 end
 
 # Parse docstrings attached by a space or single newline
-# "doc" foo  ==>  (macrocall core_@doc "doc" foo)
 #
 # flisp: parse-docstring
 function parse_docstring(ps::ParseState, down=parse_eq)
     mark = position(ps)
-    atdoc_mark = bump_invisible(ps, K"TOMBSTONE")
     down(ps)
     if peek_behind(ps).kind == K"string"
         is_doc = true
@@ -521,19 +519,18 @@ function parse_docstring(ps::ParseState, down=parse_eq)
                 is_doc = false
             else
                 # Allow a single newline
-                # "doc" \n foo ==> (macrocall core_@doc (string "doc") foo)
+                # "doc" \n foo ==> (doc (string "doc") foo)
                 bump(ps, TRIVIA_FLAG) # NewlineWs
             end
         else
-            # "doc" foo    ==> (macrocall core_@doc (string "doc") foo)
-            # "doc $x" foo ==> (macrocall core_@doc (string "doc " x) foo)
+            # "doc" foo    ==> (doc (string "doc") foo)
+            # "doc $x" foo ==> (doc (string "doc " x) foo)
             # Allow docstrings with embedded trailing whitespace trivia
-            # """\n doc\n """ foo ==> (macrocall core_@doc (string-s "doc\n") foo)
+            # """\n doc\n """ foo ==> (doc (string-s "doc\n") foo)
         end
         if is_doc
-            reset_node!(ps, atdoc_mark, kind=K"core_@doc")
             down(ps)
-            emit(ps, mark, K"macrocall")
+            emit(ps, mark, K"doc")
         end
     end
 end
@@ -2003,7 +2000,7 @@ function parse_resword(ps::ParseState)
             parse_unary_prefix(ps)
         end
         # module A \n a \n b \n end  ==>  (module true A (block a b))
-        # module A \n "x"\na \n end  ==>  (module true A (block (core_@doc (string "x") a)))
+        # module A \n "x"\na \n end  ==>  (module true A (block (doc (string "x") a)))
         parse_block(ps, parse_docstring)
         bump_closing_token(ps, K"end")
         emit(ps, mark, K"module")

src/syntax_tree.jl

@@ -117,8 +117,6 @@ function SyntaxNode(source::SourceFile, raw::GreenNode{SyntaxHead}, position::In
             Symbol("@$(normalize_identifier(val_str))_str")
         elseif k == K"CmdMacroName"
             Symbol("@$(normalize_identifier(val_str))_cmd")
-        elseif k == K"core_@doc"
-            Symbol("core_@doc")
         elseif k == K"core_@cmd"
             Symbol("core_@cmd")
         elseif is_syntax_kind(raw)

test/expr.jl

@@ -295,4 +295,11 @@
                  Expr(:block, LineNumberNode(1), :w),
                  Expr(:block, LineNumberNode(1), :z))
     end
+
+    @testset "Core.@doc" begin
+        @test parse(Expr, "\"x\" f") ==
+            Expr(:macrocall, GlobalRef(Core, Symbol("@doc")), LineNumberNode(1), "x", :f)
+        @test parse(Expr, "\n\"x\" f") ==
+            Expr(:macrocall, GlobalRef(Core, Symbol("@doc")), LineNumberNode(2), "x", :f)
+    end
 end

test/parser.jl

@@ -59,7 +59,7 @@ tests = [
         "a;b;c"   => "(toplevel a b c)"
         "a;;;b;;" => "(toplevel a b)"
         """ "x" a ; "y" b """ =>
-            """(toplevel (macrocall core_@doc (string "x") a) (macrocall core_@doc (string "y") b))"""
+            """(toplevel (doc (string "x") a) (doc (string "y") b))"""
         "x y"  =>  "x (error-t y)"
     ],
     JuliaSyntax.parse_eq => [
@@ -487,7 +487,7 @@ tests = [
         "module do \n end"  =>  "(module true (error (do)) (block))"
         "module \$A end"    =>  "(module true (\$ A) (block))"
         "module A \n a \n b \n end"  =>  "(module true A (block a b))"
-        """module A \n "x"\na\n end""" => """(module true A (block (macrocall core_@doc (string "x") a)))"""
+        """module A \n "x"\na\n end""" => """(module true A (block (doc (string "x") a)))"""
         # export
         "export a"   =>  "(export a)"  => Expr(:export, :a)
         "export @a"  =>  "(export @a)" => Expr(:export, Symbol("@a"))
@@ -912,11 +912,11 @@ tests = [
         """ "notdoc" ]        """ => "(string \"notdoc\")"
         """ "notdoc" \n]      """ => "(string \"notdoc\")"
         """ "notdoc" \n\n foo """ => "(string \"notdoc\")"
-        """ "doc" \n foo      """ => """(macrocall core_@doc (string "doc") foo)"""
-        """ "doc" foo         """ => """(macrocall core_@doc (string "doc") foo)"""
-        """ "doc \$x" foo     """ => """(macrocall core_@doc (string "doc " x) foo)"""
+        """ "doc" \n foo      """ => """(doc (string "doc") foo)"""
+        """ "doc" foo         """ => """(doc (string "doc") foo)"""
+        """ "doc \$x" foo     """ => """(doc (string "doc " x) foo)"""
         # Allow docstrings with embedded trailing whitespace trivia
-        "\"\"\"\n doc\n \"\"\" foo"  => """(macrocall core_@doc (string-s "doc\\n") foo)"""
+        "\"\"\"\n doc\n \"\"\" foo"  => """(doc (string-s "doc\\n") foo)"""
     ],
 ]