Make generators visible in the parse tree

- Replace `_comprehension_clause` with `generator`
- Place the _bracket_form inside the generator (instead of having the
  comprehension clause as a sibling in the parse tree).

Author: savq

grammar.js

@@ -166,7 +166,6 @@ module.exports = grammar({
     [$.juxtaposition_expression, $._expression],
     [$.matrix_row, $.comprehension_expression], // Comprehensions with newlines
     [$.parenthesized_expression, $.tuple_expression],
-    [$._bracket_form, $.tuple_expression],
   ],
 
   supertypes: $ => [
@@ -534,11 +533,16 @@ module.exports = grammar({
       '[',
       $._bracket_form,
       optional($._terminator),
-      $._comprehension_clause,
+      $.for_clause,
+      repeat(choice(
+        $.for_clause,
+        $.if_clause,
+      )),
       ']',
     )),
 
-    _comprehension_clause: $ => seq(
+    generator: $ => seq(
+      $._bracket_form,
       $.for_clause,
       repeat(choice(
         $.for_clause,
@@ -586,16 +590,18 @@ module.exports = grammar({
     ),
 
     parenthesized_expression: $ => prec.dynamic(1, parenthesize(
-      sep1($._semicolon, $._bracket_form),
-      optional($._comprehension_clause),
+      sep1($._semicolon, choice(
+        $._bracket_form,
+        $.generator,
+      )),
       optional($._semicolon),
     )),
 
     tuple_expression: $ => parenthesize(
       optional($._semicolon),
       sep(choice(',', $._semicolon), choice(
         $._bracket_form,
-        seq($._expression, $._comprehension_clause),
+        $.generator,
       )),
       optional(','),
     ),

test/corpus/collections.txt

@@ -158,11 +158,7 @@ comprehension array collections
 ]
 UInt[b(c, e) for c in d for e in f]
 
-f(1, 2, i for i in iter)
-(b(c, e) for c in d, e = 5 if e)
-
 ---
-
 (source_file
   (comprehension_expression
     (identifier)
@@ -177,19 +173,30 @@ f(1, 2, i for i in iter)
     (comprehension_expression
       (call_expression (identifier) (argument_list (identifier) (identifier)))
       (for_clause (for_binding (identifier) (operator) (identifier)))
-      (for_clause (for_binding (identifier) (operator) (identifier)))))
+      (for_clause (for_binding (identifier) (operator) (identifier))))))
+
+
+========================================
+generators
+========================================
+
+f(1, 2, i for i in iter)
+(b(c, e) for c in d, e = 5 if e)
 
+---
+(source_file
   (call_expression
-    (identifier)
-    (argument_list
-      (integer_literal)
-      (integer_literal)
       (identifier)
-      (for_clause (for_binding (identifier) (operator) (identifier)))))
-  (parenthesized_expression
-    (call_expression (identifier) (argument_list (identifier) (identifier)))
-    (for_clause
-      (for_binding (identifier) (operator) (identifier))
-      (for_binding (identifier) (operator) (integer_literal)))
-    (if_clause (identifier))))
-
+      (argument_list
+        (integer_literal)
+        (integer_literal)
+        (generator
+          (identifier)
+          (for_clause (for_binding (identifier) (operator) (identifier))))))
+    (parenthesized_expression
+      (generator
+        (call_expression (identifier) (argument_list (identifier) (identifier)))
+        (for_clause
+          (for_binding (identifier) (operator) (identifier))
+          (for_binding (identifier) (operator) (integer_literal)))
+        (if_clause (identifier)))))