diff --git a/detect_unused.sh b/detect_unused.sh new file mode 100644 index 0000000..93e1100 --- /dev/null +++ b/detect_unused.sh @@ -0,0 +1 @@ +vulture ./parser ./main.py ./fuzz.py ./test --ignore-decorators="@_register_autowalk_expr*" --exclude="parser/astgen/ast_print.py" diff --git a/parser/astgen/ast_node.py b/parser/astgen/ast_node.py index e743aef..2d9541d 100644 --- a/parser/astgen/ast_node.py +++ b/parser/astgen/ast_node.py @@ -6,10 +6,10 @@ from ..common import HasRegion, StrRegion __all__ = [ - "AstNode", "AstProgramNode", "VarDeclType", "AstDeclNode", "AstRepeat", - "AstIf", "AstWhile", "AstAssign", "AstAugAssign", "AstDefine", "AstNumber", - "AstString", "AstAnyName", "AstIdent", "AstAttrName", "AstAttribute", - "AstItem", "AstCall", "AstOp", "AstBinOp", "AstUnaryOp", + "AstNode", "AstProgramNode", "VarDeclScope", "VarType", "AstDeclNode", + "AstRepeat", "AstIf", "AstWhile", "AstAssign", "AstAugAssign", "AstDefine", + "AstNumber", "AstString", "AstAnyName", "AstIdent", "AstAttrName", + "AstAttribute", "AstItem", "AstCall", "AstOp", "AstBinOp", "AstUnaryOp", ] @@ -27,15 +27,21 @@ class AstProgramNode(AstNode): # region ---- ---- -class VarDeclType(Enum): +class VarDeclScope(Enum): LET = 'let' GLOBAL = 'global' +class VarType(Enum): + VARIABLE = 'variable' + LIST = 'list' + + @dataclass class AstDeclNode(AstNode): name = 'var_decl' - type: VarDeclType + scope: VarDeclScope + type: VarType decls: list[tuple[AstIdent, AstNode | None]] diff --git a/parser/astgen/astgen.py b/parser/astgen/astgen.py index 161c5ee..b1f5ccf 100644 --- a/parser/astgen/astgen.py +++ b/parser/astgen/astgen.py @@ -26,8 +26,7 @@ WhileBlock, RepeatBlock, DefineNode, - LetNode, - GlobalNode, + DeclNode, AssignOpNode, ) @@ -101,10 +100,8 @@ def _walk_program(self, root: ProgramNode): def _walk_smt(self, smt: AnyNode) -> list[AstNode]: if isinstance(smt, NopNode): return [] - elif isinstance(smt, LetNode): - return self._walk_var_decl(smt, VarDeclType.LET) - elif isinstance(smt, GlobalNode): - return self._walk_var_decl(smt, VarDeclType.GLOBAL) + elif isinstance(smt, DeclNode): + return self._walk_var_decl(smt) elif isinstance(smt, RepeatBlock): return [AstRepeat(smt.region, self._walk_expr(smt.count), self._walk_block(smt.block))] @@ -136,11 +133,15 @@ def _walk_smt(self, smt: AnyNode) -> list[AstNode]: f"expressions have no side-effect so are not allowed at " f"the root level.", smt.region) - def _walk_var_decl(self, smt: LetNode | GlobalNode, decl_tp: VarDeclType): + def _walk_var_decl(self, smt: DeclNode): decls = [(self._walk_ident(d.ident), None if d.value is None else self._walk_expr(d.value)) - for d in smt.decls] - return [AstDeclNode(smt.region, decl_tp, decls)] + for d in smt.decl_list.decls] + scope = (VarDeclScope.LET if isinstance(smt.decl_scope, DeclScope_Let) + else VarDeclScope.GLOBAL) + tp = (VarType.LIST if isinstance(smt.decl_type, DeclType_List) + else VarType.VARIABLE) + return [AstDeclNode(smt.region, scope, tp, decls)] def _walk_assign_left(self, lhs: AnyNode) -> AstNode: if isinstance(lhs, IdentNode): diff --git a/parser/cst/named_node.py b/parser/cst/named_node.py index dbdb9cb..81dd0cd 100644 --- a/parser/cst/named_node.py +++ b/parser/cst/named_node.py @@ -4,7 +4,7 @@ from typing import TYPE_CHECKING, overload, Sequence, cast, Literal from .base_node import Leaf, AnyNode, Node -from ..common import StrRegion +from ..common import StrRegion, HasRegion if TYPE_CHECKING: from ..tokens import Token @@ -26,7 +26,7 @@ def __post_init__(self): # noinspection PyMethodOverriding @classmethod - def of(cls, token: Token, parent: Node | None = None): + def of(cls, token: HasRegion, parent: Node | None = None): return cls(token.region, parent) @@ -49,7 +49,7 @@ def __init__(self, region: StrRegion, parent: Node | None = None, # noinspection PyMethodOverriding @classmethod - def of(cls, token: Token, parent: Node | None = None): + def of(cls, token: HasRegion, parent: Node | None = None): return cls(token.region, parent) @@ -70,7 +70,7 @@ def __post_init__(self): # noinspection PyMethodOverriding @classmethod - def of(cls, token: Token, children: list[AnyNode] | None = None, + def of(cls, token: HasRegion, children: list[AnyNode] | None = None, parent: Node | None = None): return cls(token.region, parent, children or []) diff --git a/parser/cst/nodes.py b/parser/cst/nodes.py index ec43eea..1d6c7ef 100644 --- a/parser/cst/nodes.py +++ b/parser/cst/nodes.py @@ -6,19 +6,38 @@ from .named_node import (NamedLeafCls, NamedNodeCls, NamedSizedNodeCls, register_corresponding_token) -__all__ = [ - "ProgramNode", "NumberNode", "StringNode", "AnyNameLeaf", "IdentNode", - "AttrNameNode", "AutocatNode", "GetattrNode", "GetitemNode", "ParenNode", - "CallNode", "CallArgs", "OperatorNode", "UnaryOpNode", "UPlusNode", - "UMinusNode", "NotNode", "BinOpNode", "AddNode", "SubNode", "MulNode", - "DivNode", "ModNode", "PowNode", "ConcatNode", "AndNode", "OrNode", +__all__ = [ # Keep these sorted by category + "ProgramNode", "AnyNullNode", + # Atoms + "NumberNode", "StringNode", "AnyNameLeaf", "IdentNode", "AttrNameNode", + "AutocatNode", # autocat is sorta atom-ish (it is in AST but not in CST?) + # Item chains + "GetattrNode", "GetitemNode", "ParenNode", "CallNode", "CallArgs", + # Operators + "OperatorNode", + "UnaryOpNode", "UPlusNode", "UMinusNode", "NotNode", # Unary operators + # Binary operators + "BinOpNode", "AddNode", "SubNode", "MulNode", "DivNode", "ModNode", + "PowNode", "ConcatNode", "AndNode", "OrNode", + # Comparisons "ComparisonNode", "EqNode", "NeqNode", "LtNode", "LeNode", "GtNode", - "GeNode", "NopNode", "BlockNode", "ConditionalBlock", "IfBlock", - "ElseIfBlock", "ElseBlock", "NullElseBlock", "WhileBlock", "RepeatBlock", - "DefineNode", "ArgsDeclNode", "ArgDeclNode", "DeclItemNode", "LetNode", - "GlobalNode", "AssignOpNode", "AssignNode", "AddEqNode", "SubEqNode", - "MulEqNode", "DivEqNode", "ModEqNode", "PowEqNode", "ConcatEqNode", - "AndEqNode", "OrEqNode", + "GeNode", + + "NopNode", + # Blocks + "BlockNode", "WhileBlock", "RepeatBlock", + # If blocks & related infrastructure + "ConditionalBlock", "IfBlock", "ElseIfBlock", "ElseBlock", "NullElseBlock", + # Function Definitions + "DefineNode", "ArgsDeclNode", "ArgDeclNode", + # Variable declarations & related infrastructure + "DeclNode", "DeclItemsList", "DeclItemNode", + "DeclScopeNode", "DeclScope_Let", "DeclScope_Global", + "DeclTypeNode", "DeclType_Variable", "DeclType_List", + # Assignment (regular and augmented) + "AssignOpNode", "AssignNode", + "AddEqNode", "SubEqNode", "MulEqNode", "DivEqNode", "ModEqNode", + "PowEqNode", "ConcatEqNode", "AndEqNode", "OrEqNode", ] @@ -30,6 +49,11 @@ def statements(self): return self.children +class AnyNullNode(NamedLeafCls): + """For nodes whose presence denotes an absence of something in the syntax""" + size = 0 # Name should be set by subclasses + + # region ---- Expressions ---- @register_corresponding_token class NumberNode(NamedLeafCls): @@ -305,9 +329,8 @@ def block(self): return checked_cast(BlockNode, self.children[0]) -class NullElseBlock(NamedLeafCls): +class NullElseBlock(AnyNullNode): name = 'else_null' - size = 0 class WhileBlock(NamedSizedNodeCls): @@ -375,6 +398,7 @@ def ident(self): # endregion +# region ---- Variable Decls ---- class DeclItemNode(NamedNodeCls): name = 'decl_item' # 1 or 2 (name and optional value) @@ -389,20 +413,58 @@ def value(self): return self.children[1] -class LetNode(NamedNodeCls): - name = 'let_decl' # Varargs +class DeclScopeNode(AnyNullNode): + pass + + +# noinspection PyPep8Naming +class DeclScope_Let(DeclScopeNode): + name = 'scope__let' + + +# noinspection PyPep8Naming +class DeclScope_Global(DeclScopeNode): + name = 'scope__global' + + +class DeclTypeNode(AnyNullNode): + pass + + +# noinspection PyPep8Naming +class DeclType_Variable(DeclTypeNode): + name = 'decl_type__variable' + + +# noinspection PyPep8Naming +class DeclType_List(DeclTypeNode): + name = 'decl_type__list' + + +class DeclNode(NamedSizedNodeCls): + name = 'var_decl' + size = 3 # scope, type (value/list), decl_list @property - def decls(self): - return cast(list[DeclItemNode], self.children) + def decl_scope(self): + return checked_cast(DeclScopeNode, self.children[0]) + @property + def decl_type(self): + return checked_cast(DeclTypeNode, self.children[1]) + + @property + def decl_list(self): + return cast(DeclItemsList, self.children[2]) -class GlobalNode(NamedNodeCls): - name = 'global_decl' # Varargs + +class DeclItemsList(NamedNodeCls): + name = 'decl_list' @property def decls(self): return cast(list[DeclItemNode], self.children) +# endregion # region ---- Assignment-ops ---- diff --git a/parser/cst/treegen.py b/parser/cst/treegen.py index bc8e301..4afca79 100644 --- a/parser/cst/treegen.py +++ b/parser/cst/treegen.py @@ -100,10 +100,8 @@ def _parse_smt(self, idx: int) -> tuple[AnyNode, int]: smt, idx = self._parse_while(idx) elif self.matches(idx, KwdM('repeat')): smt, idx = self._parse_repeat(idx) - elif self.matches(idx, (KwdM('global'), IdentNameToken)): - smt, idx = self._parse_global(idx) - elif self.matches(idx, (KwdM('let'), IdentNameToken)): - smt, idx = self._parse_let(idx) + elif self.matches(idx, KwdM('let')) or self.matches(idx, KwdM('global')): + smt, idx = self._parse_decl(idx) # Could be faster ^^ elif self.matches(idx, SemicolonToken): smt = NopNode(self.tok_region(idx, idx + 1)) idx += 1 @@ -132,37 +130,43 @@ def _parse_expr_or_assign(self, idx: int) -> tuple[AnyNode, int]: raise self.err(f"Expected semicolon at end of expr, " f"got {self[idx].name}", self[idx]) - def _parse_let(self, start: int) -> tuple[AnyNode, int]: - idx = start - assert self.matches(idx, KwdM('let')) - idx += 1 - items, idx = self._parse_decl_item_list(idx) + def _parse_decl(self, idx: int): # e.g. global [] foo=bar, ...; + scope, idx = self._parse_decl_scope(idx) # ~~~~~^ ^^ ~~^~~~~~~~~~ + tp_node, idx = self._parse_decl_sqb_or(idx) # ~/ | + decl_items, idx = self._parse_decl_item_list(idx) # ~~~/ if not self.matches(idx, SemicolonToken): raise self.err(f"Expected ';' or ',' after decl_item," f" got {self[idx].name}", self[idx]) idx += 1 - return LetNode(self.tok_region(start, idx), None, items), idx + return self.node_from_children(DeclNode, [scope, tp_node, decl_items]), idx + + def _parse_decl_scope(self, idx: int) -> tuple[AnyNode, int]: + if self.matches(idx, KwdM('let')): + return DeclScope_Let.of(self[idx]), idx + 1 + if self.matches(idx, KwdM('global')): + return DeclScope_Global.of(self[idx]), idx + 1 + assert 0, "Unknown decl scope" + + def _parse_decl_sqb_or(self, idx: int) -> tuple[AnyNode, int]: + if not isinstance(self[idx], LSqBracket): + loc = self[idx - 1].region.end # Have to give region, so say char after 'let' + return DeclType_Variable(StrRegion(loc, loc)), idx + sqb_start = idx + idx += 1 + if not isinstance(self[idx], RSqBracket): + raise self.err("Expected ']' after '[' in list decl (eg. 'let[] a;')", self[idx]) + idx += 1 + return DeclType_List(self.tok_region(sqb_start, idx)), idx - def _parse_decl_item_list(self, idx): + def _parse_decl_item_list(self, idx: int) -> tuple[AnyNode, int]: items = [] while True: - glob, idx = self._parse_decl_item(idx) - items.append(glob) + item, idx = self._parse_decl_item(idx) + items.append(item) if not self.matches(idx, CommaToken): break idx += 1 - return items, idx - - def _parse_global(self, start: int) -> tuple[AnyNode, int]: - idx = start - assert self.matches(idx, KwdM('global')) - idx += 1 - items, idx = self._parse_decl_item_list(idx) - if not self.matches(idx, SemicolonToken): - raise self.err(f"Expected ';' or ',' after decl_item," - f" got {self[idx].name}", self[idx]) - idx += 1 - return GlobalNode(self.tok_region(start, idx), None, items), idx + return self.node_from_children(DeclItemsList, items), idx def _parse_decl_item(self, start: int) -> tuple[AnyNode, int]: idx = start diff --git a/test/.snapshots/test_astgen.txt b/test/.snapshots/test_astgen.txt index 0bcb67b..ade8d10 100644 --- a/test/.snapshots/test_astgen.txt +++ b/test/.snapshots/test_astgen.txt @@ -1,5 +1,5 @@ -"2","14","45","93","122","184","192","204" -"TestAstGen::test_op_node:0","TestAstGen::test_nop_node:0","TestAstGen::test_aug_assign:0","TestAstGen::test_getattr:0","TestAstGen::test_while_block:0","TestAstGen::test_autocat:0","TestAstGen::test_string_basic:0","TestAstGen::test_unaries:0" +"2","14","45","93","122","184","192","204","254" +"TestAstGen::test_op_node:0","TestAstGen::test_nop_node:0","TestAstGen::test_aug_assign:0","TestAstGen::test_getattr:0","TestAstGen::test_while_block:0","TestAstGen::test_autocat:0","TestAstGen::test_string_basic:0","TestAstGen::test_unaries:0","TestAstGen::test_decl:0" AstProgramNode(StrRegion(0, 6), [ AstAssign(StrRegion(0, 5), @@ -251,4 +251,71 @@ AstProgramNode(StrRegion(0, 25), ) ) ] +) +AstProgramNode(StrRegion(0, 86), + [ + AstDeclNode(StrRegion(0, 13), + VarDeclScope.LET, + VarType.VARIABLE, + [ + ( + AstIdent(StrRegion(4, 5), 'a'), + None + ), + ( + AstIdent(StrRegion(6, 7), 'b'), + AstBinOp(StrRegion(8, 11), + '+', + AstNumber(StrRegion(8, 9), 1), + AstNumber(StrRegion(10, 11), 1) + ) + ), + ( + AstIdent(StrRegion(12, 13), 'c'), + None + ) + ] + ), + AstDeclNode(StrRegion(15, 37), + VarDeclScope.GLOBAL, + VarType.VARIABLE, + [ + ( + AstIdent(StrRegion(22, 23), 'd'), + AstString(StrRegion(26, 34), 'STRING') + ), + ( + AstIdent(StrRegion(36, 37), 'e'), + None + ) + ] + ), + AstDeclNode(StrRegion(39, 69), + VarDeclScope.LET, + VarType.LIST, + [ + ( + AstIdent(StrRegion(45, 55), 'local_list'), + AstCall(StrRegion(56, 62), + AstIdent(StrRegion(56, 60), 'list'), + [] + ) + ), + ( + AstIdent(StrRegion(64, 69), 'other'), + None + ) + ] + ), + AstDeclNode(StrRegion(71, 85), + VarDeclScope.GLOBAL, + VarType.LIST, + [ + ( + AstIdent(StrRegion(80, 85), 'STACK'), + None + ) + ] + ) + ] ) \ No newline at end of file diff --git a/test/.snapshots/test_main_examples.txt b/test/.snapshots/test_main_examples.txt index 5b54501..c19652c 100644 --- a/test/.snapshots/test_main_examples.txt +++ b/test/.snapshots/test_main_examples.txt @@ -1,4 +1,4 @@ -"2","90","175","666","1042" +"2","90","179","670","1094" "TestMain::test_example_0:tokens","TestMain::test_example_0:cst","TestMain::test_example_1:tokens","TestMain::test_example_1:cst","TestMain::test_example_1:ast" [IdentNameToken(region=StrRegion(start=0, end=3)), WhitespaceToken(region=StrRegion(start=3, end=4)), @@ -89,16 +89,20 @@ WhitespaceToken(region=StrRegion(start=144, end=145)), EofToken(region=StrRegion(start=145, end=145))] ProgramNode(StrRegion(0, 144), [ - LetNode(StrRegion(0, 18), [ - DeclItemNode(StrRegion(4, 17), [ - IdentNode(StrRegion(4, 5)), - AddNode(StrRegion(8, 17), [ - MulNode(StrRegion(8, 11), [ - NumberNode(StrRegion(8, 9)), - NumberNode(StrRegion(10, 11)) - ]), - UMinusNode(StrRegion(15, 17), [ - NumberNode(StrRegion(16, 17)) + DeclNode(StrRegion(0, 17), [ + DeclScope_Let(StrRegion(0, 3)), + DeclType_Variable(StrRegion(3, 3)), + DeclItemsList(StrRegion(4, 17), [ + DeclItemNode(StrRegion(4, 17), [ + IdentNode(StrRegion(4, 5)), + AddNode(StrRegion(8, 17), [ + MulNode(StrRegion(8, 11), [ + NumberNode(StrRegion(8, 9)), + NumberNode(StrRegion(10, 11)) + ]), + UMinusNode(StrRegion(15, 17), [ + NumberNode(StrRegion(16, 17)) + ]) ]) ]) ]) @@ -665,19 +669,27 @@ ProgramNode(StrRegion(0, 144), [ WhitespaceToken(region=StrRegion(start=1685, end=1688)), EofToken(region=StrRegion(start=1688, end=1688))] ProgramNode(StrRegion(0, 1685), [ - GlobalNode(StrRegion(0, 22), [ - DeclItemNode(StrRegion(7, 21), [ - IdentNode(StrRegion(7, 12)), - CallNode(StrRegion(15, 21), [ - IdentNode(StrRegion(15, 19)), - CallArgs(StrRegion(19, 21), []) + DeclNode(StrRegion(0, 21), [ + DeclScope_Global(StrRegion(0, 6)), + DeclType_Variable(StrRegion(6, 6)), + DeclItemsList(StrRegion(7, 21), [ + DeclItemNode(StrRegion(7, 21), [ + IdentNode(StrRegion(7, 12)), + CallNode(StrRegion(15, 21), [ + IdentNode(StrRegion(15, 19)), + CallArgs(StrRegion(19, 21), []) + ]) ]) ]) ]), - GlobalNode(StrRegion(23, 49), [ - DeclItemNode(StrRegion(30, 48), [ - IdentNode(StrRegion(30, 40)), - IdentNode(StrRegion(43, 48)) + DeclNode(StrRegion(23, 48), [ + DeclScope_Global(StrRegion(23, 29)), + DeclType_Variable(StrRegion(29, 29)), + DeclItemsList(StrRegion(30, 48), [ + DeclItemNode(StrRegion(30, 48), [ + IdentNode(StrRegion(30, 40)), + IdentNode(StrRegion(43, 48)) + ]) ]) ]), DefineNode(StrRegion(398, 508), [ @@ -724,23 +736,31 @@ ProgramNode(StrRegion(0, 1685), [ IdentNode(StrRegion(514, 521)), ArgsDeclNode(StrRegion(521, 523), []), BlockNode(StrRegion(524, 660), [ - LetNode(StrRegion(530, 555), [ - DeclItemNode(StrRegion(534, 554), [ - IdentNode(StrRegion(534, 541)), - CallNode(StrRegion(544, 554), [ - IdentNode(StrRegion(544, 547)), - CallArgs(StrRegion(547, 554), [ - IdentNode(StrRegion(548, 553)) + DeclNode(StrRegion(530, 554), [ + DeclScope_Let(StrRegion(530, 533)), + DeclType_Variable(StrRegion(533, 533)), + DeclItemsList(StrRegion(534, 554), [ + DeclItemNode(StrRegion(534, 554), [ + IdentNode(StrRegion(534, 541)), + CallNode(StrRegion(544, 554), [ + IdentNode(StrRegion(544, 547)), + CallArgs(StrRegion(547, 554), [ + IdentNode(StrRegion(548, 553)) + ]) ]) ]) ]) ]), - LetNode(StrRegion(560, 586), [ - DeclItemNode(StrRegion(564, 585), [ - IdentNode(StrRegion(564, 568)), - GetitemNode(StrRegion(571, 585), [ - IdentNode(StrRegion(571, 576)), - IdentNode(StrRegion(577, 584)) + DeclNode(StrRegion(560, 585), [ + DeclScope_Let(StrRegion(560, 563)), + DeclType_Variable(StrRegion(563, 563)), + DeclItemsList(StrRegion(564, 585), [ + DeclItemNode(StrRegion(564, 585), [ + IdentNode(StrRegion(564, 568)), + GetitemNode(StrRegion(571, 585), [ + IdentNode(StrRegion(571, 576)), + IdentNode(StrRegion(577, 584)) + ]) ]) ]) ]), @@ -773,23 +793,31 @@ ProgramNode(StrRegion(0, 1685), [ IdentNode(StrRegion(666, 675)), ArgsDeclNode(StrRegion(675, 677), []), BlockNode(StrRegion(678, 972), [ - LetNode(StrRegion(797, 822), [ - DeclItemNode(StrRegion(801, 821), [ - IdentNode(StrRegion(801, 808)), - CallNode(StrRegion(811, 821), [ - IdentNode(StrRegion(811, 814)), - CallArgs(StrRegion(814, 821), [ - IdentNode(StrRegion(815, 820)) + DeclNode(StrRegion(797, 821), [ + DeclScope_Let(StrRegion(797, 800)), + DeclType_Variable(StrRegion(800, 800)), + DeclItemsList(StrRegion(801, 821), [ + DeclItemNode(StrRegion(801, 821), [ + IdentNode(StrRegion(801, 808)), + CallNode(StrRegion(811, 821), [ + IdentNode(StrRegion(811, 814)), + CallArgs(StrRegion(814, 821), [ + IdentNode(StrRegion(815, 820)) + ]) ]) ]) ]) ]), - LetNode(StrRegion(827, 853), [ - DeclItemNode(StrRegion(831, 852), [ - IdentNode(StrRegion(831, 835)), - GetitemNode(StrRegion(838, 852), [ - IdentNode(StrRegion(838, 843)), - IdentNode(StrRegion(844, 851)) + DeclNode(StrRegion(827, 852), [ + DeclScope_Let(StrRegion(827, 830)), + DeclType_Variable(StrRegion(830, 830)), + DeclItemsList(StrRegion(831, 852), [ + DeclItemNode(StrRegion(831, 852), [ + IdentNode(StrRegion(831, 835)), + GetitemNode(StrRegion(838, 852), [ + IdentNode(StrRegion(838, 843)), + IdentNode(StrRegion(844, 851)) + ]) ]) ]) ]), @@ -838,15 +866,19 @@ ProgramNode(StrRegion(0, 1685), [ IdentNode(StrRegion(978, 985)), ArgsDeclNode(StrRegion(985, 987), []), BlockNode(StrRegion(988, 1100), [ - LetNode(StrRegion(994, 1024), [ - DeclItemNode(StrRegion(998, 1023), [ - IdentNode(StrRegion(998, 1003)), - GetitemNode(StrRegion(1006, 1023), [ - IdentNode(StrRegion(1006, 1011)), - CallNode(StrRegion(1012, 1022), [ - IdentNode(StrRegion(1012, 1015)), - CallArgs(StrRegion(1015, 1022), [ - IdentNode(StrRegion(1016, 1021)) + DeclNode(StrRegion(994, 1023), [ + DeclScope_Let(StrRegion(994, 997)), + DeclType_Variable(StrRegion(997, 997)), + DeclItemsList(StrRegion(998, 1023), [ + DeclItemNode(StrRegion(998, 1023), [ + IdentNode(StrRegion(998, 1003)), + GetitemNode(StrRegion(1006, 1023), [ + IdentNode(StrRegion(1006, 1011)), + CallNode(StrRegion(1012, 1022), [ + IdentNode(StrRegion(1012, 1015)), + CallArgs(StrRegion(1015, 1022), [ + IdentNode(StrRegion(1016, 1021)) + ]) ]) ]) ]) @@ -881,34 +913,46 @@ ProgramNode(StrRegion(0, 1685), [ IdentNode(StrRegion(1106, 1117)), ArgsDeclNode(StrRegion(1117, 1119), []), BlockNode(StrRegion(1120, 1358), [ - LetNode(StrRegion(1126, 1151), [ - DeclItemNode(StrRegion(1130, 1150), [ - IdentNode(StrRegion(1130, 1137)), - CallNode(StrRegion(1140, 1150), [ - IdentNode(StrRegion(1140, 1143)), - CallArgs(StrRegion(1143, 1150), [ - IdentNode(StrRegion(1144, 1149)) + DeclNode(StrRegion(1126, 1150), [ + DeclScope_Let(StrRegion(1126, 1129)), + DeclType_Variable(StrRegion(1129, 1129)), + DeclItemsList(StrRegion(1130, 1150), [ + DeclItemNode(StrRegion(1130, 1150), [ + IdentNode(StrRegion(1130, 1137)), + CallNode(StrRegion(1140, 1150), [ + IdentNode(StrRegion(1140, 1143)), + CallArgs(StrRegion(1143, 1150), [ + IdentNode(StrRegion(1144, 1149)) + ]) ]) ]) ]) ]), - LetNode(StrRegion(1156, 1182), [ - DeclItemNode(StrRegion(1160, 1181), [ - IdentNode(StrRegion(1160, 1164)), - GetitemNode(StrRegion(1167, 1181), [ - IdentNode(StrRegion(1167, 1172)), - IdentNode(StrRegion(1173, 1180)) + DeclNode(StrRegion(1156, 1181), [ + DeclScope_Let(StrRegion(1156, 1159)), + DeclType_Variable(StrRegion(1159, 1159)), + DeclItemsList(StrRegion(1160, 1181), [ + DeclItemNode(StrRegion(1160, 1181), [ + IdentNode(StrRegion(1160, 1164)), + GetitemNode(StrRegion(1167, 1181), [ + IdentNode(StrRegion(1167, 1172)), + IdentNode(StrRegion(1173, 1180)) + ]) ]) ]) ]), - LetNode(StrRegion(1187, 1217), [ - DeclItemNode(StrRegion(1191, 1216), [ - IdentNode(StrRegion(1191, 1195)), - GetitemNode(StrRegion(1198, 1216), [ - IdentNode(StrRegion(1198, 1203)), - SubNode(StrRegion(1204, 1215), [ - IdentNode(StrRegion(1204, 1211)), - NumberNode(StrRegion(1214, 1215)) + DeclNode(StrRegion(1187, 1216), [ + DeclScope_Let(StrRegion(1187, 1190)), + DeclType_Variable(StrRegion(1190, 1190)), + DeclItemsList(StrRegion(1191, 1216), [ + DeclItemNode(StrRegion(1191, 1216), [ + IdentNode(StrRegion(1191, 1195)), + GetitemNode(StrRegion(1198, 1216), [ + IdentNode(StrRegion(1198, 1203)), + SubNode(StrRegion(1204, 1215), [ + IdentNode(StrRegion(1204, 1211)), + NumberNode(StrRegion(1214, 1215)) + ]) ]) ]) ]) @@ -957,23 +1001,31 @@ ProgramNode(StrRegion(0, 1685), [ IdentNode(StrRegion(1364, 1372)), ArgsDeclNode(StrRegion(1372, 1374), []), BlockNode(StrRegion(1375, 1671), [ - LetNode(StrRegion(1455, 1480), [ - DeclItemNode(StrRegion(1459, 1479), [ - IdentNode(StrRegion(1459, 1466)), - CallNode(StrRegion(1469, 1479), [ - IdentNode(StrRegion(1469, 1472)), - CallArgs(StrRegion(1472, 1479), [ - IdentNode(StrRegion(1473, 1478)) + DeclNode(StrRegion(1455, 1479), [ + DeclScope_Let(StrRegion(1455, 1458)), + DeclType_Variable(StrRegion(1458, 1458)), + DeclItemsList(StrRegion(1459, 1479), [ + DeclItemNode(StrRegion(1459, 1479), [ + IdentNode(StrRegion(1459, 1466)), + CallNode(StrRegion(1469, 1479), [ + IdentNode(StrRegion(1469, 1472)), + CallArgs(StrRegion(1472, 1479), [ + IdentNode(StrRegion(1473, 1478)) + ]) ]) ]) ]) ]), - LetNode(StrRegion(1485, 1507), [ - DeclItemNode(StrRegion(1489, 1506), [ - IdentNode(StrRegion(1489, 1493)), - GetitemNode(StrRegion(1496, 1506), [ - IdentNode(StrRegion(1496, 1501)), - IdentNode(StrRegion(1502, 1505)) + DeclNode(StrRegion(1485, 1506), [ + DeclScope_Let(StrRegion(1485, 1488)), + DeclType_Variable(StrRegion(1488, 1488)), + DeclItemsList(StrRegion(1489, 1506), [ + DeclItemNode(StrRegion(1489, 1506), [ + IdentNode(StrRegion(1489, 1493)), + GetitemNode(StrRegion(1496, 1506), [ + IdentNode(StrRegion(1496, 1501)), + IdentNode(StrRegion(1502, 1505)) + ]) ]) ]) ]), @@ -1042,8 +1094,9 @@ ProgramNode(StrRegion(0, 1685), [ ]) AstProgramNode(StrRegion(0, 1685), [ - AstDeclNode(StrRegion(0, 22), - VarDeclType.GLOBAL, + AstDeclNode(StrRegion(0, 21), + VarDeclScope.GLOBAL, + VarType.VARIABLE, [ ( AstIdent(StrRegion(7, 12), 'stack'), @@ -1054,8 +1107,9 @@ AstProgramNode(StrRegion(0, 1685), ) ] ), - AstDeclNode(StrRegion(23, 49), - VarDeclType.GLOBAL, + AstDeclNode(StrRegion(23, 48), + VarDeclScope.GLOBAL, + VarType.VARIABLE, [ ( AstIdent(StrRegion(30, 40), 'COUNT_REFS'), @@ -1105,8 +1159,9 @@ AstProgramNode(StrRegion(0, 1685), AstIdent(StrRegion(514, 521), 'ROT_TWO'), [], [ - AstDeclNode(StrRegion(530, 555), - VarDeclType.LET, + AstDeclNode(StrRegion(530, 554), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(534, 541), 'tos_idx'), @@ -1119,8 +1174,9 @@ AstProgramNode(StrRegion(0, 1685), ) ] ), - AstDeclNode(StrRegion(560, 586), - VarDeclType.LET, + AstDeclNode(StrRegion(560, 585), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(564, 568), 'tos0'), @@ -1162,8 +1218,9 @@ AstProgramNode(StrRegion(0, 1685), AstIdent(StrRegion(666, 675), 'ROT_THREE'), [], [ - AstDeclNode(StrRegion(797, 822), - VarDeclType.LET, + AstDeclNode(StrRegion(797, 821), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(801, 808), 'tos_idx'), @@ -1176,8 +1233,9 @@ AstProgramNode(StrRegion(0, 1685), ) ] ), - AstDeclNode(StrRegion(827, 853), - VarDeclType.LET, + AstDeclNode(StrRegion(827, 852), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(831, 835), 'tos0'), @@ -1237,8 +1295,9 @@ AstProgramNode(StrRegion(0, 1685), AstIdent(StrRegion(978, 985), 'DUP_TOP'), [], [ - AstDeclNode(StrRegion(994, 1024), - VarDeclType.LET, + AstDeclNode(StrRegion(994, 1023), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(998, 1003), 'value'), @@ -1281,8 +1340,9 @@ AstProgramNode(StrRegion(0, 1685), AstIdent(StrRegion(1106, 1117), 'DUP_TOP_TWO'), [], [ - AstDeclNode(StrRegion(1126, 1151), - VarDeclType.LET, + AstDeclNode(StrRegion(1126, 1150), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(1130, 1137), 'tos_idx'), @@ -1295,8 +1355,9 @@ AstProgramNode(StrRegion(0, 1685), ) ] ), - AstDeclNode(StrRegion(1156, 1182), - VarDeclType.LET, + AstDeclNode(StrRegion(1156, 1181), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(1160, 1164), 'tos0'), @@ -1307,8 +1368,9 @@ AstProgramNode(StrRegion(0, 1685), ) ] ), - AstDeclNode(StrRegion(1187, 1217), - VarDeclType.LET, + AstDeclNode(StrRegion(1187, 1216), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(1191, 1195), 'tos1'), @@ -1365,8 +1427,9 @@ AstProgramNode(StrRegion(0, 1685), AstIdent(StrRegion(1364, 1372), 'ROT_FOUR'), [], [ - AstDeclNode(StrRegion(1455, 1480), - VarDeclType.LET, + AstDeclNode(StrRegion(1455, 1479), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(1459, 1466), 'tos_idx'), @@ -1379,8 +1442,9 @@ AstProgramNode(StrRegion(0, 1685), ) ] ), - AstDeclNode(StrRegion(1485, 1507), - VarDeclType.LET, + AstDeclNode(StrRegion(1485, 1506), + VarDeclScope.LET, + VarType.VARIABLE, [ ( AstIdent(StrRegion(1489, 1493), 'tos0'), diff --git a/test/test_astgen.py b/test/test_astgen.py index 1a24366..c9e5230 100644 --- a/test/test_astgen.py +++ b/test/test_astgen.py @@ -46,3 +46,9 @@ def test_autocat(self): def test_unaries(self): self.assertAstMatchesSnapshot('a=+(-!b==!-+c)-+--r+(-9);') + + def test_decl(self): + self.assertAstMatchesSnapshot('let a,b=1+1,c;\n' + 'global d = "STRING", e;\n' + 'let[] local_list=list(), other;\n' + 'global[] STACK;') diff --git a/test/test_integration/.snapshots/test_empty_expr.txt b/test/test_integration/.snapshots/test_empty_expr.txt index 3b11d7c..652c8c2 100644 --- a/test/test_integration/.snapshots/test_empty_expr.txt +++ b/test/test_integration/.snapshots/test_empty_expr.txt @@ -1,11 +1,15 @@ "2" "MyTestCase::test_empty_expr_issue_04:0" ProgramNode(StrRegion(0, 9), [ - LetNode(StrRegion(0, 8), [ - DeclItemNode(StrRegion(4, 7), [ - IdentNode(StrRegion(4, 5)), - NumberNode(StrRegion(6, 7)) + DeclNode(StrRegion(0, 7), [ + DeclScope_Let(StrRegion(0, 3)), + DeclType_Variable(StrRegion(3, 3)), + DeclItemsList(StrRegion(4, 7), [ + DeclItemNode(StrRegion(4, 7), [ + IdentNode(StrRegion(4, 5)), + NumberNode(StrRegion(6, 7)) + ]) ]) ]), NopNode(StrRegion(8, 9)) -]) +]) \ No newline at end of file diff --git a/test/test_integration/.snapshots/test_treegen.txt b/test/test_integration/.snapshots/test_treegen.txt index ff7195f..05e83cd 100644 --- a/test/test_integration/.snapshots/test_treegen.txt +++ b/test/test_integration/.snapshots/test_treegen.txt @@ -1,5 +1,5 @@ -"2","45","93","105","114","121","135","154","181","198","225","242","269","297","315","348" -"TreeGenTest::test_item_chain:0","TreeGenTest::test_fn_call_in_lvalue:0","TreeGenTest::test_aug_assign:0","TreeGenTest::test__mod_supported:0","TreeGenTest::test_decl_no_value:0","TestFunctionDecl::test_no_params:0","TestFunctionDecl::test_one_param:0","TestFunctionDecl::test_two_param:0","TestBlocks::test_while:0","TestBlocks::test_while:1","TestBlocks::test_repeat:0","TestBlocks::test_repeat:1","TestBlocks::test_else_if_else:0","TestBlocks::test_else_if_else:1","TestBlocks::test_else_if_else:2","TestAutocat::test_autocat:0" +"2","45","93","105","114","125","139","158","185","202","229","246","273","301","319","352","371","447" +"TreeGenTest::test_item_chain:0","TreeGenTest::test_fn_call_in_lvalue:0","TreeGenTest::test_aug_assign:0","TreeGenTest::test__mod_supported:0","TreeGenTest::test_decl_no_value:0","TestFunctionDecl::test_no_params:0","TestFunctionDecl::test_one_param:0","TestFunctionDecl::test_two_param:0","TestBlocks::test_while:0","TestBlocks::test_while:1","TestBlocks::test_repeat:0","TestBlocks::test_repeat:1","TestBlocks::test_else_if_else:0","TestBlocks::test_else_if_else:1","TestBlocks::test_else_if_else:2","TestAutocat::test_autocat:0","TreeGenTest::test_decl_multiple:0","TreeGenTest::test_decl:0" ProgramNode(StrRegion(0, 43), [ AssignNode(StrRegion(0, 42), [ GetitemNode(StrRegion(0, 21), [ @@ -113,9 +113,13 @@ ProgramNode(StrRegion(0, 6), [ ]) ]) ProgramNode(StrRegion(0, 6), [ - LetNode(StrRegion(0, 6), [ - DeclItemNode(StrRegion(4, 5), [ - IdentNode(StrRegion(4, 5)) + DeclNode(StrRegion(0, 5), [ + DeclScope_Let(StrRegion(0, 3)), + DeclType_Variable(StrRegion(3, 3)), + DeclItemsList(StrRegion(4, 5), [ + DeclItemNode(StrRegion(4, 5), [ + IdentNode(StrRegion(4, 5)) + ]) ]) ]) ]) @@ -364,4 +368,139 @@ ProgramNode(StrRegion(0, 32), [ CallArgs(StrRegion(29, 31), []) ]) ]) +]) +ProgramNode(StrRegion(0, 80), [ + DeclNode(StrRegion(0, 32), [ + DeclScope_Let(StrRegion(0, 3)), + DeclType_Variable(StrRegion(3, 3)), + DeclItemsList(StrRegion(4, 32), [ + DeclItemNode(StrRegion(4, 5), [ + IdentNode(StrRegion(4, 5)) + ]), + DeclItemNode(StrRegion(7, 10), [ + IdentNode(StrRegion(7, 8)), + NumberNode(StrRegion(9, 10)) + ]), + DeclItemNode(StrRegion(12, 13), [ + IdentNode(StrRegion(12, 13)) + ]), + DeclItemNode(StrRegion(14, 22), [ + IdentNode(StrRegion(14, 15)), + GetitemNode(StrRegion(16, 22), [ + GetattrNode(StrRegion(16, 19), [ + IdentNode(StrRegion(16, 17)), + AttrNameNode(StrRegion(18, 19)) + ]), + NumberNode(StrRegion(20, 21)) + ]) + ]), + DeclItemNode(StrRegion(23, 28), [ + IdentNode(StrRegion(23, 24)), + StringNode(StrRegion(25, 28)) + ]), + DeclItemNode(StrRegion(29, 30), [ + IdentNode(StrRegion(29, 30)) + ]), + DeclItemNode(StrRegion(31, 32), [ + IdentNode(StrRegion(31, 32)) + ]) + ]) + ]), + DeclNode(StrRegion(34, 79), [ + DeclScope_Global(StrRegion(34, 40)), + DeclType_Variable(StrRegion(40, 40)), + DeclItemsList(StrRegion(41, 79), [ + DeclItemNode(StrRegion(41, 42), [ + IdentNode(StrRegion(41, 42)) + ]), + DeclItemNode(StrRegion(43, 48), [ + IdentNode(StrRegion(43, 44)), + UMinusNode(StrRegion(45, 48), [ + NumberNode(StrRegion(46, 48)) + ]) + ]), + DeclItemNode(StrRegion(50, 51), [ + IdentNode(StrRegion(50, 51)) + ]), + DeclItemNode(StrRegion(54, 67), [ + IdentNode(StrRegion(54, 55)), + CallNode(StrRegion(56, 67), [ + GetattrNode(StrRegion(56, 65), [ + StringNode(StrRegion(56, 59)), + AttrNameNode(StrRegion(60, 65)) + ]), + CallArgs(StrRegion(65, 67), []) + ]) + ]), + DeclItemNode(StrRegion(69, 72), [ + IdentNode(StrRegion(69, 70)), + IdentNode(StrRegion(71, 72)) + ]), + DeclItemNode(StrRegion(75, 76), [ + IdentNode(StrRegion(75, 76)) + ]), + DeclItemNode(StrRegion(78, 79), [ + IdentNode(StrRegion(78, 79)) + ]) + ]) + ]) +]) +ProgramNode(StrRegion(0, 86), [ + DeclNode(StrRegion(0, 13), [ + DeclScope_Let(StrRegion(0, 3)), + DeclType_Variable(StrRegion(3, 3)), + DeclItemsList(StrRegion(4, 13), [ + DeclItemNode(StrRegion(4, 5), [ + IdentNode(StrRegion(4, 5)) + ]), + DeclItemNode(StrRegion(6, 11), [ + IdentNode(StrRegion(6, 7)), + AddNode(StrRegion(8, 11), [ + NumberNode(StrRegion(8, 9)), + NumberNode(StrRegion(10, 11)) + ]) + ]), + DeclItemNode(StrRegion(12, 13), [ + IdentNode(StrRegion(12, 13)) + ]) + ]) + ]), + DeclNode(StrRegion(15, 37), [ + DeclScope_Global(StrRegion(15, 21)), + DeclType_Variable(StrRegion(21, 21)), + DeclItemsList(StrRegion(22, 37), [ + DeclItemNode(StrRegion(22, 34), [ + IdentNode(StrRegion(22, 23)), + StringNode(StrRegion(26, 34)) + ]), + DeclItemNode(StrRegion(36, 37), [ + IdentNode(StrRegion(36, 37)) + ]) + ]) + ]), + DeclNode(StrRegion(39, 69), [ + DeclScope_Let(StrRegion(39, 42)), + DeclType_List(StrRegion(42, 44)), + DeclItemsList(StrRegion(45, 69), [ + DeclItemNode(StrRegion(45, 62), [ + IdentNode(StrRegion(45, 55)), + CallNode(StrRegion(56, 62), [ + IdentNode(StrRegion(56, 60)), + CallArgs(StrRegion(60, 62), []) + ]) + ]), + DeclItemNode(StrRegion(64, 69), [ + IdentNode(StrRegion(64, 69)) + ]) + ]) + ]), + DeclNode(StrRegion(71, 85), [ + DeclScope_Global(StrRegion(71, 77)), + DeclType_List(StrRegion(77, 79)), + DeclItemsList(StrRegion(80, 85), [ + DeclItemNode(StrRegion(80, 85), [ + IdentNode(StrRegion(80, 85)) + ]) + ]) + ]) ]) \ No newline at end of file diff --git a/test/test_integration/test_treegen.py b/test/test_integration/test_treegen.py index e460004..6a45fab 100644 --- a/test/test_integration/test_treegen.py +++ b/test/test_integration/test_treegen.py @@ -28,6 +28,26 @@ def test__mod_supported(self): def test_decl_no_value(self): self.assertCstMatchesSnapshot('let b;') + def test_decl_multiple(self): + self.assertCstMatchesSnapshot( + 'let a, b=9, c,d=w.1[2],e="w",f,g;\n' + 'global z,y=-11, x , w="a".lower() ,v=o , u, t;') + + def test_decl(self): + self.assertCstMatchesSnapshot('let a,b=1+1,c;\n' + 'global d = "STRING", e;\n' + 'let[] local_list=list(), other;\n' + 'global[] STACK;') + + def test_decl_error(self): + err = self.assertFailsGracefullyCST('let[4] = 9;') + self.assertEqual(StrRegion(4, 5), err.region) + self.assertContains(err.msg.lower(), "expected ']' after '['") + + err = self.assertFailsGracefullyCST('let[') + self.assertEqual(StrRegion(4, 4), err.region) + self.assertContains(err.msg.lower(), "expected ']' after '['") + class TestBlocks(CommonTestCase): def test_while(self): @@ -279,7 +299,7 @@ def test__bin_op__expr(self): self.assertFailsGracefullyCST(f'a{op}b') def test__bin_op__success(self): - for op in set(BINARY_OPS): + for op in BINARY_OPS: with self.subTest(op=op): self.assertValidParseCST(f'a{op}b;')