[Tolk] Support trailing comma in various places

Trailing comma is now allowed in:
* tensors
* tuples
* function calls
* function parameters

Note, that `(5)` is not a tensor, it's just integer `5`.
With a trailing comma `(5,)` it's still `(5)`.

Author: tolk-vm

tolk-tester/tests/a6.tolk

@@ -4,7 +4,7 @@ fun f(a: int, b: int, c: int, d: int, e: int, f: int): (int, int) {
   __expect_type(D, "int");
   __expect_type(D*D, "int");
   __expect_type(calc_phi, "() -> int");
-  return (Dx/D,Dy/D);
+  return (Dx/D,Dy/D,);
 };;;;
 
 fun calc_phi(): int {
@@ -16,7 +16,7 @@ fun calc_phi(): int {
   do {
     (p,q)=(q,p+q);
   } while (q <= n); //;;
-  return mulDivRound(p, n, q);
+  return mulDivRound(p, n, q,);
 }
 
 fun calc_sqrt2(): int {
@@ -31,10 +31,10 @@ fun calc_sqrt2(): int {
   return mulDivRound(p, n, q);
 }
 
-fun calc_root(m: int) {
+fun calc_root(m: int,) {
   var base: int=1;
   repeat(70) { base *= 10; }
-  var (a, b, c) = (1,0,-m);
+  var (a, b, c) = (1,0,-m,);
   var (p1, q1, p2, q2) = (1, 0, 0, 1);
   do {
     var k: int=-1;

tolk-tester/tests/allow_post_modification.tolk

@@ -47,7 +47,7 @@ fun test_call_1(x: int): (int, int, int, int, int, int, int) {
     return foo1(x, x.`~inc`(x / 20), x, x = x * 2, x, x += 1, x);
 }
 
-fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int): (int, int, int, int, int, int, int) {
+fun foo2(x1: int, x2: int, x3456: (int, int, int, int), x7: int, ): (int, int, int, int, int, int, int) {
     var (x3: int, x4: int, x5: int, x6: int) = x3456;
     return (x1, x2, x3, x4, x5, x6, x7);
 }

tolk-tester/tests/assignment-tests.tolk

@@ -5,7 +5,7 @@ fun extractFromTypedTuple(params: [int]) {
 
 @method_id(101)
 fun test101(x: int) {
-    var params = [x];
+    var params = [x, ];
     return extractFromTypedTuple(params);
 }
 
@@ -38,7 +38,7 @@ fun getTensor_1X(x: int) {
 }
 fun getTuple_12() {
     callOrder.tuplePush(110);
-    return [1, 2];
+    return [1, 2, ];
 }
 fun getTuple_1X(x: int) {
     callOrder.tuplePush(111);
@@ -133,7 +133,7 @@ fun test108() {
 fun test109() {
     callOrder = createEmptyTuple();
     var x = 0;
-    [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x)];
+    [getTuple_12().0, getTuple_1X(x = getIntValue5()).1, getTuple_1X(x += 10).0] = [getIntValue5(), getIntValue5(), getIntValueX(x), ];
     return (callOrder, x);
 }
 
@@ -203,9 +203,18 @@ fun test116() {
     return (a, b, c, d, rhs2);
 }
 
+@method_id(117)
+fun test117() {
+    var c = [((1, ), ), ];
+    __expect_type(c, "[int]");
+    c.0 = 10;
+    c.0 = (20, );
+    return c;
+}
+
 
 
-fun main(value: int) {
+fun main(value: int, ) {
     var (x: int?, y) = (autoInferIntNull(value), autoInferIntNull(value * 2));
     if (x == null && y == null) { return null; }
     return x == null || y == null ? -1 : x! + y!;
@@ -231,6 +240,7 @@ fun main(value: int) {
 @testcase | 114 |        | 13 [ 1 14 ] 1 3
 @testcase | 115 |        | [ 101 111 ] 9 9
 @testcase | 116 |        | 1 2 3 4 [ 1 2 3 4 ]
+@testcase | 117 |        | [ 20 ]
 
 
 @fif_codegen

tolk-tester/tests/inference-tests.tolk

@@ -30,6 +30,15 @@ fun test1(x: int, y: int) {
         __expect_type(beginCell(), "builder");
         __expect_type(beginCell().endCell(), "cell");
     }
+    __expect_type((1, 2), "(int, int)");
+    __expect_type((1, 2, ), "(int, int)");
+    __expect_type((1, ), "int");
+    __expect_type((((1,)),), "int");
+    __expect_type([1, 2], "[int, int]");
+    __expect_type([1, 2, ], "[int, int]");
+    __expect_type([1, ], "[int]");
+    __expect_type([((1, ), ), ], "[int]");
+    __expect_type([[[1, ], ], ], "[[[int]]]");
 }
 
 fun test2(x: int, y: bool) {

tolk-tester/tests/no-spaces.tolk

@@ -3,11 +3,11 @@ const int10:int=10;
 fun just10(): int { return int10; }
 fun eq(v: int): int { return`v`; }
 
-@method_id(101) fun `get_-1`  (): int {return-1;}
-@method_id(102) fun `get_--1` (): int {return--1;}
-@method_id(103) fun `get_---1`(): int {return---1;}
-@method_id(104) fun `get_+++1`(): int {return+++1;}
-@method_id(105) fun `get_+-+1`(): int {return+-+1;}
+@method_id(101,) fun `get_-1`  (): int {return-1;}
+@method_id(102,) fun `get_--1` (): int {return--1;}
+@method_id(103,) fun `get_---1`(): int {return---1;}
+@method_id(104,) fun `get_+++1`(): int {return+++1;}
+@method_id(105,) fun `get_+-+1`(): int {return+-+1;}
 
 global `some()var`:int;
 

tolk-tester/tests/nullable-tensors.tolk

@@ -415,6 +415,13 @@ fun test135() {
         t1!.0 == null, t2!.0 == null, e1!.1.0 == null, e1!.1.1 == null, e2!.1.0 == null, e2!.1.1 == null);
 }
 
+@method_id(136, )
+fun test136(x: int?, ) {
+    var t1 = (x, );     // it's not a tensor with 1 element
+    __expect_type(t1, "int?", );
+    return ((t1, t1 == null, ), );   // testing trailing comma everywhere :)
+}
+
 
 
 fun main(){}
@@ -466,6 +473,8 @@ fun main(){}
 @testcase | 133 |      | 60
 @testcase | 134 |      | 11 21 -1
 @testcase | 135 |      | [ 10 ] [ (null) ] (null) 777 10 -1 (null) -1 (null) 0 777 10 -1 (null) -1 (null) 0 777 0 0 -1 0 0 -1 0 0 -1 777 0 -1 0 0 -1 0
+@testcase | 136 | 9    | 9 0
+@testcase | 136 | null | (null) -1
 
 @fif_codegen
 """

tolk/ast-from-tokens.cpp

@@ -236,6 +236,9 @@ static V<ast_parameter_list> parse_parameter_list(Lexer& lex) {
     params.push_back(parse_parameter(lex, true));
     while (lex.tok() == tok_comma) {
       lex.next();
+      if (lex.tok() == tok_clpar) {     // trailing comma
+        break;
+      }
       params.push_back(parse_parameter(lex, false));
     }
   }
@@ -266,6 +269,9 @@ static V<ast_argument_list> parse_argument_list(Lexer& lex) {
     args.push_back(parse_argument(lex));
     while (lex.tok() == tok_comma) {
       lex.next();
+      if (lex.tok() == tok_clpar) {   // trailing comma
+        break;
+      }
       args.push_back(parse_argument(lex));
     }
   }
@@ -311,9 +317,15 @@ static AnyExprV parse_expr100(Lexer& lex) {
       std::vector<AnyExprV> items(1, first);
       while (lex.tok() == tok_comma) {
         lex.next();
+        if (lex.tok() == tok_clpar) {   // trailing comma
+          break;
+        }
         items.emplace_back(parse_expr(lex));
       }
       lex.expect(tok_clpar, "`)`");
+      if (items.size() == 1) {      // we can reach here for 1 element with trailing comma: `(item, )`
+        return items[0];            // then just return item, not a 1-element tensor,
+      }                             // since 1-element tensors won't be type compatible with item's type
       return createV<ast_tensor>(loc, std::move(items));
     }
     case tok_opbracket: {
@@ -325,6 +337,9 @@ static AnyExprV parse_expr100(Lexer& lex) {
       std::vector<AnyExprV> items(1, parse_expr(lex));
       while (lex.tok() == tok_comma) {
         lex.next();
+        if (lex.tok() == tok_clbracket) {   // trailing comma
+          break;
+        }
         items.emplace_back(parse_expr(lex));
       }
       lex.expect(tok_clbracket, "`]`");
@@ -952,6 +967,9 @@ static V<ast_annotation> parse_annotation(Lexer& lex) {
     args.push_back(parse_expr(lex));
     while (lex.tok() == tok_comma) {
       lex.next();
+      if (lex.tok() == tok_clpar) {   // trailing comma
+        break;
+      }
       args.push_back(parse_expr(lex));
     }
     lex.expect(tok_clpar, "`)`");