Merge pull request #52 from MarcellPerger1/add-list-decl

Add list declarations

Author: MarcellPerger1

detect_unused.sh

@@ -0,0 +1 @@
+vulture ./parser ./main.py ./fuzz.py ./test --ignore-decorators="@_register_autowalk_expr*" --exclude="parser/astgen/ast_print.py"

parser/astgen/ast_node.py

@@ -6,10 +6,10 @@
 from ..common import HasRegion, StrRegion
 
 __all__ = [
-    "AstNode", "AstProgramNode", "VarDeclType", "AstDeclNode", "AstRepeat",
-    "AstIf", "AstWhile", "AstAssign", "AstAugAssign", "AstDefine", "AstNumber",
-    "AstString", "AstAnyName", "AstIdent", "AstAttrName", "AstAttribute",
-    "AstItem", "AstCall", "AstOp", "AstBinOp", "AstUnaryOp",
+    "AstNode", "AstProgramNode", "VarDeclScope", "VarType", "AstDeclNode",
+    "AstRepeat", "AstIf", "AstWhile", "AstAssign", "AstAugAssign", "AstDefine",
+    "AstNumber", "AstString", "AstAnyName", "AstIdent", "AstAttrName",
+    "AstAttribute", "AstItem", "AstCall", "AstOp", "AstBinOp", "AstUnaryOp",
 ]
 
 
@@ -27,15 +27,21 @@ class AstProgramNode(AstNode):
 
 
 # region ---- <Statements> ----
-class VarDeclType(Enum):
+class VarDeclScope(Enum):
     LET = 'let'
     GLOBAL = 'global'
 
 
+class VarType(Enum):
+    VARIABLE = 'variable'
+    LIST = 'list'
+
+
 @dataclass
 class AstDeclNode(AstNode):
     name = 'var_decl'
-    type: VarDeclType
+    scope: VarDeclScope
+    type: VarType
     decls: list[tuple[AstIdent, AstNode | None]]
 
 

parser/astgen/astgen.py

@@ -26,8 +26,7 @@
     WhileBlock,
     RepeatBlock,
     DefineNode,
-    LetNode,
-    GlobalNode,
+    DeclNode,
     AssignOpNode,
 )
 
@@ -101,10 +100,8 @@ def _walk_program(self, root: ProgramNode):
     def _walk_smt(self, smt: AnyNode) -> list[AstNode]:
         if isinstance(smt, NopNode):
             return []
-        elif isinstance(smt, LetNode):
-            return self._walk_var_decl(smt, VarDeclType.LET)
-        elif isinstance(smt, GlobalNode):
-            return self._walk_var_decl(smt, VarDeclType.GLOBAL)
+        elif isinstance(smt, DeclNode):
+            return self._walk_var_decl(smt)
         elif isinstance(smt, RepeatBlock):
             return [AstRepeat(smt.region, self._walk_expr(smt.count),
                               self._walk_block(smt.block))]
@@ -136,11 +133,15 @@ def _walk_smt(self, smt: AnyNode) -> list[AstNode]:
                 f"expressions have no side-effect so are not allowed at "
                 f"the root level.", smt.region)
 
-    def _walk_var_decl(self, smt: LetNode | GlobalNode, decl_tp: VarDeclType):
+    def _walk_var_decl(self, smt: DeclNode):
         decls = [(self._walk_ident(d.ident),
                   None if d.value is None else self._walk_expr(d.value))
-                 for d in smt.decls]
-        return [AstDeclNode(smt.region, decl_tp, decls)]
+                 for d in smt.decl_list.decls]
+        scope = (VarDeclScope.LET if isinstance(smt.decl_scope, DeclScope_Let)
+                 else VarDeclScope.GLOBAL)
+        tp = (VarType.LIST if isinstance(smt.decl_type, DeclType_List)
+              else VarType.VARIABLE)
+        return [AstDeclNode(smt.region, scope, tp, decls)]
 
     def _walk_assign_left(self, lhs: AnyNode) -> AstNode:
         if isinstance(lhs, IdentNode):

parser/cst/named_node.py

@@ -4,7 +4,7 @@
 from typing import TYPE_CHECKING, overload, Sequence, cast, Literal
 
 from .base_node import Leaf, AnyNode, Node
-from ..common import StrRegion
+from ..common import StrRegion, HasRegion
 
 if TYPE_CHECKING:
     from ..tokens import Token
@@ -26,7 +26,7 @@ def __post_init__(self):
 
     # noinspection PyMethodOverriding
     @classmethod
-    def of(cls, token: Token, parent: Node | None = None):
+    def of(cls, token: HasRegion, parent: Node | None = None):
         return cls(token.region, parent)
 
 
@@ -49,7 +49,7 @@ def __init__(self, region: StrRegion, parent: Node | None = None,
 
     # noinspection PyMethodOverriding
     @classmethod
-    def of(cls, token: Token, parent: Node | None = None):
+    def of(cls, token: HasRegion, parent: Node | None = None):
         return cls(token.region, parent)
 
 
@@ -70,7 +70,7 @@ def __post_init__(self):
 
     # noinspection PyMethodOverriding
     @classmethod
-    def of(cls, token: Token, children: list[AnyNode] | None = None,
+    def of(cls, token: HasRegion, children: list[AnyNode] | None = None,
            parent: Node | None = None):
         return cls(token.region, parent, children or [])
 

parser/cst/nodes.py

@@ -6,19 +6,38 @@
 from .named_node import (NamedLeafCls, NamedNodeCls, NamedSizedNodeCls,
                          register_corresponding_token)
 
-__all__ = [
-    "ProgramNode", "NumberNode", "StringNode", "AnyNameLeaf", "IdentNode",
-    "AttrNameNode", "AutocatNode", "GetattrNode", "GetitemNode", "ParenNode",
-    "CallNode", "CallArgs", "OperatorNode", "UnaryOpNode", "UPlusNode",
-    "UMinusNode", "NotNode", "BinOpNode", "AddNode", "SubNode", "MulNode",
-    "DivNode", "ModNode", "PowNode", "ConcatNode", "AndNode", "OrNode",
+__all__ = [  # Keep these sorted by category
+    "ProgramNode", "AnyNullNode",
+    # Atoms
+    "NumberNode", "StringNode", "AnyNameLeaf", "IdentNode", "AttrNameNode",
+    "AutocatNode",  # autocat is sorta atom-ish (it is in AST but not in CST?)
+    # Item chains
+    "GetattrNode", "GetitemNode", "ParenNode", "CallNode", "CallArgs",
+    # Operators
+    "OperatorNode",
+    "UnaryOpNode", "UPlusNode", "UMinusNode", "NotNode",  # Unary operators
+    # Binary operators
+    "BinOpNode", "AddNode", "SubNode", "MulNode", "DivNode", "ModNode",
+    "PowNode", "ConcatNode", "AndNode", "OrNode",
+    # Comparisons
     "ComparisonNode", "EqNode", "NeqNode", "LtNode", "LeNode", "GtNode",
-    "GeNode", "NopNode", "BlockNode", "ConditionalBlock", "IfBlock",
-    "ElseIfBlock", "ElseBlock", "NullElseBlock", "WhileBlock", "RepeatBlock",
-    "DefineNode", "ArgsDeclNode", "ArgDeclNode", "DeclItemNode", "LetNode",
-    "GlobalNode", "AssignOpNode", "AssignNode", "AddEqNode", "SubEqNode",
-    "MulEqNode", "DivEqNode", "ModEqNode", "PowEqNode", "ConcatEqNode",
-    "AndEqNode", "OrEqNode",
+    "GeNode",
+
+    "NopNode",
+    # Blocks
+    "BlockNode", "WhileBlock", "RepeatBlock",
+    # If blocks & related infrastructure
+    "ConditionalBlock", "IfBlock", "ElseIfBlock", "ElseBlock", "NullElseBlock",
+    # Function Definitions
+    "DefineNode", "ArgsDeclNode", "ArgDeclNode",
+    # Variable declarations & related infrastructure
+    "DeclNode", "DeclItemsList", "DeclItemNode",
+    "DeclScopeNode", "DeclScope_Let", "DeclScope_Global",
+    "DeclTypeNode", "DeclType_Variable", "DeclType_List",
+    # Assignment (regular and augmented)
+    "AssignOpNode", "AssignNode",
+    "AddEqNode", "SubEqNode", "MulEqNode", "DivEqNode", "ModEqNode",
+    "PowEqNode", "ConcatEqNode", "AndEqNode", "OrEqNode",
 ]
 
 
@@ -30,6 +49,11 @@ def statements(self):
         return self.children
 
 
+class AnyNullNode(NamedLeafCls):
+    """For nodes whose presence denotes an absence of something in the syntax"""
+    size = 0  # Name should be set by subclasses
+
+
 # region ---- Expressions ----
 @register_corresponding_token
 class NumberNode(NamedLeafCls):
@@ -305,9 +329,8 @@ def block(self):
         return checked_cast(BlockNode, self.children[0])
 
 
-class NullElseBlock(NamedLeafCls):
+class NullElseBlock(AnyNullNode):
     name = 'else_null'
-    size = 0
 
 
 class WhileBlock(NamedSizedNodeCls):
@@ -375,6 +398,7 @@ def ident(self):
 # endregion
 
 
+# region ---- Variable Decls ----
 class DeclItemNode(NamedNodeCls):
     name = 'decl_item'  # 1 or 2 (name and optional value)
 
@@ -389,20 +413,58 @@ def value(self):
         return self.children[1]
 
 
-class LetNode(NamedNodeCls):
-    name = 'let_decl'  # Varargs
+class DeclScopeNode(AnyNullNode):
+    pass
+
+
+# noinspection PyPep8Naming
+class DeclScope_Let(DeclScopeNode):
+    name = 'scope__let'
+
+
+# noinspection PyPep8Naming
+class DeclScope_Global(DeclScopeNode):
+    name = 'scope__global'
+
+
+class DeclTypeNode(AnyNullNode):
+    pass
+
+
+# noinspection PyPep8Naming
+class DeclType_Variable(DeclTypeNode):
+    name = 'decl_type__variable'
+
+
+# noinspection PyPep8Naming
+class DeclType_List(DeclTypeNode):
+    name = 'decl_type__list'
+
+
+class DeclNode(NamedSizedNodeCls):
+    name = 'var_decl'
+    size = 3  # scope, type (value/list), decl_list
 
     @property
-    def decls(self):
-        return cast(list[DeclItemNode], self.children)
+    def decl_scope(self):
+        return checked_cast(DeclScopeNode, self.children[0])
 
+    @property
+    def decl_type(self):
+        return checked_cast(DeclTypeNode, self.children[1])
+
+    @property
+    def decl_list(self):
+        return cast(DeclItemsList, self.children[2])
 
-class GlobalNode(NamedNodeCls):
-    name = 'global_decl'  # Varargs
+
+class DeclItemsList(NamedNodeCls):
+    name = 'decl_list'
 
     @property
     def decls(self):
         return cast(list[DeclItemNode], self.children)
+# endregion
 
 
 # region ---- Assignment-ops ----

parser/cst/treegen.py

@@ -100,10 +100,8 @@ def _parse_smt(self, idx: int) -> tuple[AnyNode, int]:
             smt, idx = self._parse_while(idx)
         elif self.matches(idx, KwdM('repeat')):
             smt, idx = self._parse_repeat(idx)
-        elif self.matches(idx, (KwdM('global'), IdentNameToken)):
-            smt, idx = self._parse_global(idx)
-        elif self.matches(idx, (KwdM('let'), IdentNameToken)):
-            smt, idx = self._parse_let(idx)
+        elif self.matches(idx, KwdM('let')) or self.matches(idx, KwdM('global')):
+            smt, idx = self._parse_decl(idx)  # Could be faster ^^
         elif self.matches(idx, SemicolonToken):
             smt = NopNode(self.tok_region(idx, idx + 1))
             idx += 1
@@ -132,37 +130,43 @@ def _parse_expr_or_assign(self, idx: int) -> tuple[AnyNode, int]:
         raise self.err(f"Expected semicolon at end of expr, "
                        f"got {self[idx].name}", self[idx])
 
-    def _parse_let(self, start: int) -> tuple[AnyNode, int]:
-        idx = start
-        assert self.matches(idx, KwdM('let'))
-        idx += 1
-        items, idx = self._parse_decl_item_list(idx)
+    def _parse_decl(self, idx: int):         # e.g. global [] foo=bar, ...;
+        scope, idx = self._parse_decl_scope(idx)  # ~~~~~^ ^^ ~~^~~~~~~~~~
+        tp_node, idx = self._parse_decl_sqb_or(idx)     # ~/    |
+        decl_items, idx = self._parse_decl_item_list(idx)  # ~~~/
         if not self.matches(idx, SemicolonToken):
             raise self.err(f"Expected ';' or ',' after decl_item,"
                            f" got {self[idx].name}", self[idx])
         idx += 1
-        return LetNode(self.tok_region(start, idx), None, items), idx
+        return self.node_from_children(DeclNode, [scope, tp_node, decl_items]), idx
+
+    def _parse_decl_scope(self, idx: int) -> tuple[AnyNode, int]:
+        if self.matches(idx, KwdM('let')):
+            return DeclScope_Let.of(self[idx]), idx + 1
+        if self.matches(idx, KwdM('global')):
+            return DeclScope_Global.of(self[idx]), idx + 1
+        assert 0, "Unknown decl scope"
+
+    def _parse_decl_sqb_or(self, idx: int) -> tuple[AnyNode, int]:
+        if not isinstance(self[idx], LSqBracket):
+            loc = self[idx - 1].region.end  # Have to give region, so say char after 'let'
+            return DeclType_Variable(StrRegion(loc, loc)), idx
+        sqb_start = idx
+        idx += 1
+        if not isinstance(self[idx], RSqBracket):
+            raise self.err("Expected ']' after '[' in list decl (eg. 'let[] a;')", self[idx])
+        idx += 1
+        return DeclType_List(self.tok_region(sqb_start, idx)), idx
 
-    def _parse_decl_item_list(self, idx):
+    def _parse_decl_item_list(self, idx: int) -> tuple[AnyNode, int]:
         items = []
         while True:
-            glob, idx = self._parse_decl_item(idx)
-            items.append(glob)
+            item, idx = self._parse_decl_item(idx)
+            items.append(item)
             if not self.matches(idx, CommaToken):
                 break
             idx += 1
-        return items, idx
-
-    def _parse_global(self, start: int) -> tuple[AnyNode, int]:
-        idx = start
-        assert self.matches(idx, KwdM('global'))
-        idx += 1
-        items, idx = self._parse_decl_item_list(idx)
-        if not self.matches(idx, SemicolonToken):
-            raise self.err(f"Expected ';' or ',' after decl_item,"
-                           f" got {self[idx].name}", self[idx])
-        idx += 1
-        return GlobalNode(self.tok_region(start, idx), None, items), idx
+        return self.node_from_children(DeclItemsList, items), idx
 
     def _parse_decl_item(self, start: int) -> tuple[AnyNode, int]:
         idx = start