Parse docstrings within structs as K"doc" (#511)

Julia's ecosystem (including Base.Docs and flisp lowering) assumes that
strings within `struct` definitions are per-field docstrings, but the
flisp parser doesn't handle these - they are only recognized when the
struct itself has a docstring and are processed by the `@doc` macro
recursing into the struct's internals. For example, the following
doesn't result in any docs attached to `A`.

```julia
struct A
    "x_docs"
    x

    "y_docs"
    y
end
```

This change adds `K"doc"` node parsing to the insides of a struct,
making the semantics clearer in the parser tree and making it possible
to address this problems in the future within JuliaLowering.

Also ensure that the `Expr` form is unaffected by this change.

Author: c42f

src/expr.jl

@@ -466,6 +466,19 @@ function _internal_node_to_Expr(source, srcrange, head, childranges, childheads,
         headsym = :call
         pushfirst!(args, :*)
     elseif k == K"struct"
+        @assert args[2].head == :block
+        orig_fields = args[2].args
+        fields = Expr(:block)
+        for field in orig_fields
+            if @isexpr(field, :macrocall) && field.args[1] == GlobalRef(Core, Symbol("@doc"))
+                # @doc macro calls don't occur within structs, in Expr form.
+                push!(fields.args, field.args[3])
+                push!(fields.args, field.args[4])
+            else
+                push!(fields.args, field)
+            end
+        end
+        args[2] = fields
         pushfirst!(args, has_flags(head, MUTABLE_FLAG))
     elseif k == K"importpath"
         headsym = :.

src/parser.jl

@@ -536,9 +536,9 @@ end
 # Parse docstrings attached by a space or single newline
 #
 # flisp: parse-docstring
-function parse_docstring(ps::ParseState)
+function parse_docstring(ps::ParseState, down=parse_eq)
     mark = position(ps)
-    parse_eq(ps)
+    down(ps)
     if peek_behind(ps).kind == K"string"
         is_doc = true
         k = peek(ps)
@@ -563,7 +563,7 @@ function parse_docstring(ps::ParseState)
             # """\n doc\n """ foo ==> (doc (string-s "doc\n") foo)
         end
         if is_doc
-            parse_eq(ps)
+            down(ps)
             emit(ps, mark, K"doc")
         end
     end
@@ -1947,7 +1947,7 @@ function parse_resword(ps::ParseState)
         @check peek(ps) == K"struct"
         bump(ps, TRIVIA_FLAG)
         parse_subtype_spec(ps)
-        parse_block(ps, parse_struct_field)
+        parse_block(ps, ps1->parse_docstring(ps1, parse_struct_field))
         bump_closing_token(ps, K"end")
         emit(ps, mark, K"struct", is_mut ? MUTABLE_FLAG : EMPTY_FLAGS)
     elseif word == K"primitive"

test/expr.jl

@@ -727,6 +727,9 @@
                  Expr(:block, LineNumberNode(2), :a, LineNumberNode(3), :b))
         @test parsestmt("struct A const a end", version=v"1.8") ==
             Expr(:struct, false, :A, Expr(:block, LineNumberNode(1), Expr(:const, :a)))
+
+        @test parsestmt("struct A \n \"doc\" \n a end") ==
+            Expr(:struct, false, :A, Expr(:block, LineNumberNode(2), "doc", :a))
     end
 
     @testset "export" begin

test/parser.jl

@@ -502,6 +502,7 @@ tests = [
         # struct
         "struct A <: B \n a::X \n end" =>  "(struct (<: A B) (block (::-i a X)))"
         "struct A \n a \n b \n end"    =>  "(struct A (block a b))"
+        "struct A \n \"doca\" \n a \n \"docb\" \n b \n end"    =>  "(struct A (block (doc (string \"doca\") a) (doc (string \"docb\") b)))"
         "mutable struct A end"         =>  "(struct-mut A (block))"
         ((v=v"1.8",), "struct A const a end") => "(struct A (block (const a)))"
         ((v=v"1.7",), "struct A const a end") => "(struct A (block (error (const a))))"