Merge pull request #60 from MarcellPerger1/add-list-literals

Add list literals, fix pow bug

Author: MarcellPerger1

main_example_1.st

@@ -1,4 +1,4 @@
-global[] stack = list();
+global[] stack = [];
 global COUNT_REFS = false;
 
 // opcodes from python version 3.8:

parser/astgen/ast_node.py

@@ -9,7 +9,8 @@
     "AstNode", "AstProgramNode", "VarDeclScope", "VarType", "AstDeclNode",
     "AstRepeat", "AstIf", "AstWhile", "AstAssign", "AstAugAssign", "AstDefine",
     "AstNumber", "AstString", "AstAnyName", "AstIdent", "AstAttrName",
-    "AstAttribute", "AstItem", "AstCall", "AstOp", "AstBinOp", "AstUnaryOp",
+    "AstListLiteral", "AstAttribute", "AstItem", "AstCall", "AstOp", "AstBinOp",
+    "AstUnaryOp",
 ]
 
 
@@ -129,6 +130,12 @@ class AstAttrName(AstAnyName):
     name = 'attr'
 
 
+@dataclass
+class AstListLiteral(AstNode):
+    name = 'list'
+    items: list[AstNode]
+
+
 @dataclass
 class AstAttribute(AstNode):
     name = '.'

parser/astgen/astgen.py

@@ -195,6 +195,13 @@ def _walk_attr_name(self, attr_name: AttrNameNode) -> AstAttrName:
         return AstAttrName(
             attr_name.region, self.node_str(attr_name, intern=True))
 
+    @_register_autowalk_expr
+    def _walk_list_literal(self, ls: ListNode):
+        # For now it is UB to use this anywhere but in variable decls
+        #  (until proper syntax desugar-ing for literal arrays is done).
+        # Anyway, for now the codegen/typechecker/desugar step will check this
+        return AstListLiteral(ls.region, [self._walk_expr(i) for i in ls.items])
+
     @_register_autowalk_expr
     def _walk_getattr(self, node: GetattrNode) -> AstAttribute:
         return AstAttribute(node.region, self._walk_expr(node.target),

parser/cst/cstgen.py

@@ -412,8 +412,32 @@ def _parse_parens_or(self, idx: int) -> tuple[AnyNode, int]:
             idx = self._expect_cls_consume(
                 idx, RParToken, f"Expected ')' at end of expr, got {self[idx].name}")
             return ParenNode(self.tok_region(start, idx), None, [inner]), idx
+        elif isinstance(self[idx], LSqBracket):
+            return self._parse_list_literal(idx)
         return self._parse_atom_or_autocat(idx)
 
+    def _parse_list_literal(self, idx: int) -> tuple[AnyNode, int]:
+        start = idx
+        assert self.matches(idx, LSqBracket)
+        idx += 1
+        if self.matches(idx, RSqBracket):
+            idx += 1  # simple case, no args
+            return ListNode(self.tok_region(start, idx)), idx
+        arg1, idx = self._parse_expr(idx)
+        args = [arg1]
+        while not self.matches(idx, RSqBracket):
+            if not self.matches(idx, CommaToken):  # if not end, must be comma
+                raise self.err(f"Expected ',' or ']' after item in list"
+                               f" literal, got {self[idx].name}", self[idx])
+            idx += 1
+            if self.matches(idx, RSqBracket):  # trailing comma, no value
+                break
+            arg, idx = self._parse_expr(idx)
+            args.append(arg)
+        assert self.matches(idx, RSqBracket)
+        idx += 1
+        return ListNode(self.tok_region(start, idx), None, args), idx
+
     def _parse_basic_item(self, idx: int):
         left, new_idx = self._parse_parens_or(idx)
         while idx != (idx := new_idx):  # If progress made, set old to current and loop again
@@ -476,7 +500,8 @@ def _parse_pow_or(self, idx: int) -> tuple[AnyNode, int]:
         idx += 1
         # parse_unary_or -> parse_pow_or -> parse_unary_or -> parse_pow_or
         # This is right-recursion so is fine as progress will be made each call to get here
-        return self._parse_unary_or(idx)
+        right, idx = self._parse_unary_or(idx)
+        return self.node_from_children(PowNode, [leftmost, right]), idx
 
     def _parse_unary_or(self, idx: int) -> tuple[AnyNode, int]:
         unaries, idx = self._parse_unaries_into_tok_list(idx)

parser/cst/nodes.py

@@ -13,6 +13,7 @@
     "AutocatNode",  # autocat is sorta atom-ish (it is in AST but not in CST?)
     # Item chains
     "GetattrNode", "GetitemNode", "ParenNode", "CallNode", "CallArgs",
+    "ListNode",
     # Operators
     "OperatorNode",
     "UnaryOpNode", "UPlusNode", "UMinusNode", "NotNode",  # Unary operators
@@ -122,6 +123,14 @@ def contents(self):
         return self.children[0]
 
 
+class ListNode(NamedNodeCls):
+    name = 'list'  # List literal, varargs
+
+    @property
+    def items(self):
+        return self.children
+
+
 class CallNode(NamedSizedNodeCls):
     name = 'call'
     size = 2

test/common/common.py

@@ -11,7 +11,7 @@
 from parser.common.error import BaseParseError
 from parser.common.tree_print import tformat
 from parser.cst.base_node import Leaf, AnyNode, Node
-from parser.cst.cstgen import CstGen, CstParseError
+from parser.cst.cstgen import CstGen, LocatedCstError
 from parser.lexer import Tokenizer
 from parser.lexer.tokens import Token, OpToken
 from test.common.snapshottest import SnapshotTestCase
@@ -87,7 +87,7 @@ def assertValidParseCST(self, src: str):
 
     def assertFailsGracefullyCST(self, src: str):
         t = CstGen(Tokenizer(src))
-        with self.assertRaises(CstParseError) as ctx:
+        with self.assertRaises(LocatedCstError) as ctx:
             t.parse()
         return ctx.exception
 

test/test_astgen/.snapshots/test_astgen.txt

@@ -52,3 +52,12 @@ def test_decl(self):
                                       'global d = "STRING", e;\n'
                                       'let[] local_list=list(), other;\n'
                                       'global[] STACK;')
+
+    def test_list_literal_decl(self):
+        self.assertAstMatchesSnapshot('let[] loc = [5, 6.,], b, c=[];\n'
+                                      'global [] STACK = [foo(bar), 8];')
+
+    def test_list_literal_decl_paren(self):
+        self.assertAstMatchesSnapshot('let[] a = ([1]);')
+    # We only test for lists in variable decls (as them being allowed
+    # elsewhere is UB for now).

test/test_astgen/test_astgen.py

@@ -14,6 +14,9 @@ def test_autocat(self):
     def test_mod_supported(self):
         self.assertCstMatchesSnapshot('c=a%b;')
 
+    def test_pow(self):
+        self.assertCstMatchesSnapshot('a**b;')
+
 
 class TestItemChain(CommonTestCase):
     def test_item_chain(self):
@@ -24,11 +27,15 @@ def test_fn_call_in_lvalue(self):
 
     def test_empty_sqb_error(self):
         with self.assertRaises(LocatedCstError) as err:
-            CstGen(Tokenizer('v=a[]+b')).parse()
+            CstGen(Tokenizer('v=a[]+b;')).parse()
         exc = err.exception
         self.assertBetweenIncl(3, 4, exc.region.start)
         self.assertEqual(4, exc.region.end - 1)
 
+    def test_empty_getitem_comma_error(self):
+        exc = self.assertFailsGracefullyCST('v=a[1,]+b;')
+        self.assertEqual(StrRegion(5, 6), exc.region)
+
     def test_getattr__issue_09(self):
         t = Tokenizer('fn(call_arg).a;').tokenize()
         node = CstGen(t).parse()
@@ -50,6 +57,23 @@ def test_getitem__issue_09(self):
         t = Tokenizer('"a string"["key_" .. 3];').tokenize()
         node = CstGen(t).parse()
         self.assertMatchesSnapshot(node, 'after_string')
+
+    def test_list_literal_empty(self):
+        self.assertCstMatchesSnapshot('let a = [];\n'
+                                      'join([], a, [].length);')
+        self.assertFailsGracefullyCST('let b = [,];')
+
+    def test_list_literal_single_item(self):
+        self.assertCstMatchesSnapshot('["a"].method([-2**--6,]);')
+
+    def test_list_literal_multi_item(self):
+        self.assertCstMatchesSnapshot(
+            'global[] STACK = ["int", 0];\n'
+            '(STACK + ["float", .2,]).extend([6.6, 1, fn()[0], 2][-1]);\n'
+            'v+=[1,2](3);')
+        e = self.assertFailsGracefullyCST('[1;')
+        self.assertEqual(StrRegion(2, 3), e.region)
+        self.assertContains(e.msg.lower(), "after item in list")
 # endregion </Test Expressions>