Parse do blocks without desugaring the closure (#98)

The reference parser represents `do` syntax with a closure for the second
argument. However, the nested closure with `->` head is implied rather
than present in the surface syntax, which suggests this is a premature
desugaring step. Instead we emit a flatter three-argument form for `do`.

Author: c42f

README.md

@@ -462,6 +462,23 @@ The same goes for command strings which are always wrapped in `K"cmdstring"`
 regardless of whether they have multiple pieces (due to triple-quoted
 dedenting) or otherwise.
 
+### No desugaring of the closure in do blocks 
+
+The reference parser represents `do` syntax with a closure for the second
+argument. That is,
+
+```julia
+f(x) do y
+    body
+end
+```
+
+becomes `(do (call f x) (-> (tuple y) (block body)))` in the reference parser.
+
+However, the nested closure with `->` head is implied here rather than present
+in the surface syntax, which suggests this is a premature desugaring step.
+Instead we emit the flatter structure `(do (call f x) (tuple y) (block body))`.
+
 ## More about syntax kinds
 
 We generally track the type of syntax nodes with a syntax "kind", stored

src/expr.jl

@@ -218,15 +218,16 @@ function _to_expr(node::SyntaxNode, iteration_spec=false, need_linenodes=true)
         end
     elseif headsym == :module
         pushfirst!(args[3].args, loc)
-    end
-    if headsym == :inert || (headsym == :quote && length(args) == 1 &&
+    elseif headsym == :inert || (headsym == :quote && length(args) == 1 &&
                  !(a1 = only(args); a1 isa Expr || a1 isa QuoteNode ||
                    a1 isa Bool  # <- compat hack, Julia 1.4+
                   ))
         return QuoteNode(only(args))
-    else
-        return Expr(headsym, args...)
+    elseif headsym == :do
+        @check length(args) == 3
+        return Expr(:do, args[1], Expr(:->, args[2], args[3]))
     end
+    return Expr(headsym, args...)
 end
 
 Base.Expr(node::SyntaxNode) = _to_expr(node)

src/parser.jl

@@ -423,8 +423,7 @@ end
 # a \n b    ==>  (block a b)
 #
 # flisp: parse-block
-function parse_block(ps::ParseState, down=parse_eq, mark=position(ps),
-                     consume_end=false)
+function parse_block(ps::ParseState, down=parse_eq, mark=position(ps))
     parse_block_inner(ps::ParseState, down)
     emit(ps, mark, K"block")
 end
@@ -1452,10 +1451,8 @@ function parse_call_chain(ps::ParseState, mark, is_macrocall=false)
             parse_call_arglist(ps, K")", is_macrocall)
             emit(ps, mark, is_macrocall ? K"macrocall" : K"call")
             if peek(ps) == K"do"
-                # f(x) do y body end  ==>  (do (call :f :x) (-> (tuple :y) (block :body)))
-                bump(ps, TRIVIA_FLAG)
-                parse_do(ps)
-                emit(ps, mark, K"do")
+                # f(x) do y body end  ==>  (do (call :f :x) (tuple :y) (block :body))
+                parse_do(ps, mark)
             end
             if is_macrocall
                 break
@@ -2179,23 +2176,24 @@ function parse_catch(ps::ParseState)
 end
 
 # flisp: parse-do
-function parse_do(ps::ParseState)
+function parse_do(ps::ParseState, mark)
+    bump(ps, TRIVIA_FLAG) # do
     ps = normal_context(ps)
-    mark = position(ps)
+    m = position(ps)
     if peek(ps) in KSet"NewlineWs ;"
-        # f() do\nend        ==>  (do (call f) (-> (tuple) (block)))
-        # f() do ; body end  ==>  (do (call f) (-> (tuple) (block body)))
+        # f() do\nend        ==>  (do (call f) (tuple) (block))
+        # f() do ; body end  ==>  (do (call f) (tuple) (block body))
         # this trivia needs to go into the tuple due to the way position()
         # works.
         bump(ps, TRIVIA_FLAG)
     else
-        # f() do x, y\n body end  ==>  (do (call f) (-> (tuple x y) (block body)))
+        # f() do x, y\n body end  ==>  (do (call f) (tuple x y) (block body))
         parse_comma_separated(ps, parse_range)
     end
-    emit(ps, mark, K"tuple")
+    emit(ps, m, K"tuple")
     parse_block(ps)
     bump_closing_token(ps, K"end")
-    emit(ps, mark, K"->")
+    emit(ps, mark, K"do")
 end
 
 function macro_name_kind(k)

test/parser.jl

@@ -242,10 +242,10 @@ tests = [
         "f(a).g(b)" => "(call (. (call f a) (quote g)) b)"
         "\$A.@x"    =>  "(macrocall (. (\$ A) (quote @x)))"
         # do
-        "f() do\nend"         =>  "(do (call f) (-> (tuple) (block)))"
-        "f() do ; body end"   =>  "(do (call f) (-> (tuple) (block body)))"
-        "f() do x, y\n body end"  =>  "(do (call f) (-> (tuple x y) (block body)))"
-        "f(x) do y body end"  =>  "(do (call f x) (-> (tuple y) (block body)))"
+        "f() do\nend"         =>  "(do (call f) (tuple) (block))"
+        "f() do ; body end"   =>  "(do (call f) (tuple) (block body))"
+        "f() do x, y\n body end"  =>  "(do (call f) (tuple x y) (block body))"
+        "f(x) do y body end"  =>  "(do (call f x) (tuple y) (block body))"
         # Keyword arguments depend on call vs macrocall
         "foo(a=1)"  =>  "(call foo (kw a 1))"
         "@foo(a=1)" =>  "(macrocall @foo (= a 1))"