More flattened form for multiple frakentuple parameters (#133)

Here we emit `(a, b; c, d; e, f)` as

    (tuple a b (parameters c d) (parameters e f))

which should make these simpler to process, rather than using the nested
form

    (tuple a b (parameters c d parameters e f)))

Author: c42f

src/expr.jl

@@ -11,6 +11,25 @@ function is_stringchunk(node)
     return k == K"String" || k == K"CmdString"
 end
 
+function reorder_parameters!(args, params_pos)
+    p = 0
+    for i = length(args):-1:1
+        if !Meta.isexpr(args[i], :parameters)
+            break
+        end
+        p = i
+    end
+    if p == 0
+        return
+    end
+    # nest frankentuples parameters sections
+    for i = length(args)-1:-1:p
+        pushfirst!(args[i].args, pop!(args))
+    end
+    # Move parameters to args[params_pos]
+    insert!(args, params_pos, pop!(args))
+end
+
 function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
                   eq_to_kw=false, inside_dot_expr=false, inside_vect_or_braces=false)
     if !haschildren(node)
@@ -145,17 +164,11 @@ function _to_expr(node::SyntaxNode; iteration_spec=false, need_linenodes=true,
             popfirst!(args)
             headsym = Symbol("'")
         end
-        # Move parameters block to args[2]
-        if length(args) > 1 && Meta.isexpr(args[end], :parameters)
-            insert!(args, 2, args[end])
-            pop!(args)
-        end
-    elseif headsym in (:tuple, :parameters, :vect, :braces)
+        # Move parameters blocks to args[2]
+        reorder_parameters!(args, 2)
+    elseif headsym in (:tuple, :vect, :braces)
         # Move parameters blocks to args[1]
-        if length(args) > 1 && Meta.isexpr(args[end], :parameters)
-            pushfirst!(args, args[end])
-            pop!(args)
-        end
+        reorder_parameters!(args, 1)
     elseif headsym in (:try, :try_finally_catch)
         # Try children in source order:
         #   try_block catch_var catch_block else_block finally_block

src/parser.jl

@@ -2934,8 +2934,8 @@ function parse_paren(ps::ParseState, check_identifiers=true)
             # Extra credit: nested parameters and frankentuples
             # (x...;)         ==> (tuple (... x) (parameters))
             # (x...; y)       ==> (tuple (... x) (parameters y))
-            # (; a=1; b=2)    ==> (tuple (parameters (= a 1) (parameters (= b 2))))
-            # (a; b; c,d)     ==> (tuple a (parameters b (parameters c d)))
+            # (; a=1; b=2)    ==> (tuple (parameters (= a 1)) (parameters (= b 2)))
+            # (a; b; c,d)     ==> (tuple a (parameters b) (parameters c d))
             # (a=1, b=2; c=3) ==> (tuple (= a 1) (= b 2) (parameters (= c 3)))
             emit(ps, mark, K"tuple")
         elseif opts.is_block
@@ -2968,7 +2968,7 @@ end
 # syntax so the parse tree is pretty strange in these cases!  Some macros
 # probably use it though.  Example:
 #
-# (a,b=1; c,d=2; e,f=3)  ==>  (tuple a (= b 1) (parameters c (= d 2) (parameters e (= f 3))))
+# (a,b=1; c,d=2; e,f=3)  ==>  (tuple a (= b 1) (parameters c (= d 2)) (parameters e (= f 3)))
 #
 # flisp: parts of parse-paren- and parse-arglist
 function parse_brackets(after_parse::Function,
@@ -2977,12 +2977,13 @@ function parse_brackets(after_parse::Function,
                     space_sensitive=false,
                     where_enabled=true,
                     whitespace_newline=true)
-    params_marks = acquire_positions(ps.stream)
+    params_positions = acquire_positions(ps.stream)
     last_eq_before_semi = 0
     num_subexprs = 0
     num_semis = 0
     had_commas = false
     had_splat = false
+    param_start = nothing
     while true
         bump_trivia(ps)
         k = peek(ps)
@@ -2991,8 +2992,11 @@ function parse_brackets(after_parse::Function,
         elseif k == K";"
             # Start of parameters list
             # a, b; c d  ==>  a b (parameters c d)
-            push!(params_marks, position(ps))
+            if !isnothing(param_start)
+                push!(params_positions, emit(ps, param_start, K"TOMBSTONE"))
+            end
             num_semis += 1
+            param_start = position(ps)
             bump(ps, TRIVIA_FLAG)
             bump_trivia(ps)
         elseif is_closing_token(ps, k)
@@ -3025,14 +3029,17 @@ function parse_brackets(after_parse::Function,
             end
         end
     end
+    if !isnothing(param_start) && position(ps) != param_start
+        push!(params_positions, emit(ps, param_start, K"TOMBSTONE"))
+    end
     opts = after_parse(had_commas, had_splat, num_semis, num_subexprs)
     # Emit nested parameter nodes if necessary
     if opts.needs_parameters
-        for mark in Iterators.reverse(params_marks)
-            emit(ps, mark, K"parameters")
+        for pos in params_positions
+            reset_node!(ps, pos, kind=K"parameters")
         end
     end
-    release_positions(ps.stream, params_marks)
+    release_positions(ps.stream, params_positions)
     bump_closing_token(ps, closing_kind)
     return opts
 end

test/parser.jl

@@ -244,6 +244,8 @@ tests = [
         "f(a,b)"  => "(call f a b)"
         "f(a=1; b=2)" => "(call f (= a 1) (parameters (= b 2)))" =>
             Expr(:call, :f, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1))
+        "f(a; b; c)" => "(call f a (parameters b) (parameters c))" =>
+            Expr(:call, :f, Expr(:parameters, Expr(:parameters, :c), :b), :a)
         "(a=1)()" =>  "(call (= a 1))" => Expr(:call, Expr(:(=), :a, 1))
         "f (a)" => "(call f (error-t) a)"
         "f(a).g(b)" => "(call (. (call f a) (quote g)) b)"
@@ -568,13 +570,15 @@ tests = [
         "(x,y)"       =>  "(tuple x y)"
         "(x=1, y=2)"  =>  "(tuple (= x 1) (= y 2))"
         # Named tuples with initial semicolon
-        "(;)"         =>  "(tuple (parameters))"
-        "(; a=1)"     =>  "(tuple (parameters (= a 1)))"
+        "(;)"         =>  "(tuple (parameters))"         =>  Expr(:tuple, Expr(:parameters))
+        "(; a=1)"     =>  "(tuple (parameters (= a 1)))" =>  Expr(:tuple, Expr(:parameters, Expr(:kw, :a, 1)))
         # Extra credit: nested parameters and frankentuples
         "(x...; y)"       => "(tuple (... x) (parameters y))"
         "(x...;)"         => "(tuple (... x) (parameters))"
-        "(; a=1; b=2)"    => "(tuple (parameters (= a 1) (parameters (= b 2))))"
-        "(a; b; c,d)"     => "(tuple a (parameters b (parameters c d)))"
+        "(; a=1; b=2)"    => "(tuple (parameters (= a 1)) (parameters (= b 2)))" =>
+            Expr(:tuple, Expr(:parameters, Expr(:parameters, Expr(:kw, :b, 2)), Expr(:kw, :a, 1)))
+        "(a; b; c,d)"     => "(tuple a (parameters b) (parameters c d))" =>
+            Expr(:tuple, Expr(:parameters, Expr(:parameters, :c, :d), :b), :a)
         "(a=1, b=2; c=3)" => "(tuple (= a 1) (= b 2) (parameters (= c 3)))"
         # Block syntax
         "(;;)"        =>  "(block)"
@@ -668,7 +672,7 @@ tests = [
         "[x,\n y]"      =>  "(vect x y)"
         "[x\n, y]"      =>  "(vect x y)"
         "[x\n,, y]"     =>  "(vect x (error-t ✘ y))"
-        "[x,y ; z]"     =>  "(vect x y (parameters z))"
+        "[x,y ; z]"     =>  "(vect x y (parameters z))"  => Expr(:vect, Expr(:parameters, :z), :x, :y)
         "[x=1, y=2]"    =>  "(vect (= x 1) (= y 2))"
         "[x=1, ; y=2]"  =>  "(vect (= x 1) (parameters (= y 2)))"
         # parse_paren