Merge pull request #37 from MarcellPerger1/start-ast

Reduce shared trees code, new `common` directory

Author: MarcellPerger1

fuzz.py

@@ -2,7 +2,7 @@
 
 from parser.lexer.tokenizer import Tokenizer
 from parser.cst.treegen import TreeGen
-from parser.error import BaseParseError
+from parser.common.error import BaseParseError
 
 from pythonfuzz.fuzzer import Fuzzer
 import pythonfuzz.fuzzer as fuzzer_ns  # For patching pythonfuzz

main.py

@@ -4,7 +4,7 @@
 from util import readfile
 from parser.cst.treegen import TreeGen
 from parser.lexer import Tokenizer, print_tokens
-from parser.trees.tree_print import tprint
+from parser.common.tree_print import tprint
 
 
 def make_tree(src: str):

parser/common/__init__.py

@@ -0,0 +1,4 @@
+from .common import *
+from .error import *  # <^ might add some stuff so `import *`
+from .str_region import StrRegion  # <-- won't add stuff to str_region so not `import *`
+from .tree_print import *

parser/common.py parser/common/common.pyparser/common.py renamed to parser/common/common.py

@@ -1,18 +1,23 @@
 from typing import TypeAlias, Sequence
 
-from .lexer.tokens import Token
 from .str_region import StrRegion
-from .trees.base_node import AnyNode
 
-RegionUnionFlatT: TypeAlias = Token | AnyNode | StrRegion
+__all__ = ['HasRegion', 'RegionUnionArgT', 'region_union']
+
+
+class HasRegion:
+    region: StrRegion
+
+
+RegionUnionFlatT: TypeAlias = HasRegion | StrRegion
 RegionUnionArgT: TypeAlias = RegionUnionFlatT | Sequence[RegionUnionFlatT]
 
 
 def region_union(*args: RegionUnionArgT):
     regs = []
     for loc in args:
-        if isinstance(loc, (Token, AnyNode)):
-            loc = loc.region  # Token and AnyNode, both have `.region`
+        if getattr(loc, 'region', None) is not None:  # Duck-type HasRegion
+            loc = loc.region
         if isinstance(loc, StrRegion):
             regs.append(loc)
         else:

parser/error.py parser/common/error.pyparser/error.py renamed to parser/common/error.py

@@ -6,8 +6,11 @@
 from dataclasses import replace as d_replace
 from typing import Sequence
 
+from .common import HasRegion
 from .str_region import StrRegion
 
+__all__ = ['BaseParseError', 'BaseLocatedError']
+
 
 # We unconditionally add this polyfill to ensure that
 # exceptions are always part of str(exc).
@@ -67,7 +70,7 @@ class BaseParseError(_PolyfillAddNoteMixin, Exception):
     pass
 
 
-class BaseLocatedError(BaseParseError):
+class BaseLocatedError(BaseParseError, HasRegion):
     __notes__: list[str]  # but might not exist
 
     def __init__(self, msg: str, region: StrRegion, src: str):

parser/str_region.py parser/common/str_region.pyparser/str_region.py renamed to parser/common/str_region.py

@@ -2,6 +2,8 @@
 
 from dataclasses import dataclass
 
+__all__ = ['StrRegion']
+
 
 @dataclass
 class StrRegion:

parser/trees/tree_print.py parser/common/tree_print.pyparser/trees/tree_print.py renamed to parser/common/tree_print.py

@@ -4,8 +4,8 @@
 from io import StringIO
 from typing import IO, Sequence
 
-from .base_node import Leaf, AnyNode, Node
-from parser.str_region import StrRegion
+from ..cst.base_node import Leaf, AnyNode, Node
+from .str_region import StrRegion
 
 __all__ = [
     'TreePrinter', 'tree_print', 'tree_format', 'tprint', 'tformat'

parser/trees/base_node.py parser/cst/base_node.pyparser/trees/base_node.py renamed to parser/cst/base_node.py

@@ -2,12 +2,12 @@
 
 from dataclasses import dataclass, field
 
+from ..common import StrRegion, HasRegion
 from ..tokens import Token
-from ..str_region import StrRegion
 
 
 @dataclass
-class Leaf:
+class Leaf(HasRegion):
     name: str
     region: StrRegion
     parent: Node | None = None

parser/trees/named_node.py parser/cst/named_node.pyparser/trees/named_node.py renamed to parser/cst/named_node.py

@@ -4,10 +4,10 @@
 from typing import TYPE_CHECKING, overload, Sequence, cast, Literal
 
 from .base_node import Leaf, AnyNode, Node
-from parser.str_region import StrRegion
+from ..common import StrRegion
 
 if TYPE_CHECKING:
-    from parser.tokens import Token
+    from ..tokens import Token
 
 
 # NOTE: these classes only follow the Liskov substitution principle on

parser/cst/nodes.py

@@ -3,8 +3,8 @@
 from typing import cast
 
 from util import checked_cast
-from ..trees.named_node import (NamedLeafCls, NamedNodeCls, NamedSizedNodeCls,
-                                register_corresponding_token)
+from .named_node import (NamedLeafCls, NamedNodeCls, NamedSizedNodeCls,
+                         register_corresponding_token)
 
 __all__ = [
     "ProgramNode", "NumberNode", "StringNode", "AnyNameLeaf", "IdentNode",