refactor(cst): Rename TreeGen to CstGen

Author: MarcellPerger1

README.md

@@ -41,7 +41,7 @@
   - Importing from outside this project: no (we don't know what extra stuff other projects put in their module namespace that might break our code).
   - Importing internally within the project:
     - Useful when you need *everything* from a module that has many *similar* classes
-    - (e.g. importing all the nodes (`parser/cst/nodes.py`) in the CST generation code (`parser/cst/treegen.py`) is good).
+    - (e.g. importing all the nodes (`parser/cst/nodes.py`) in the CST generation code (`parser/cst/cstgen.py`) is good).
     - The threshold for 'many' here is around 10-15 classes.
     - Otherwise (if the classes/functions are unrelated (e.g. in `utils.py`), or you don't need all/most of them, or the module has few classes), just list them out.
     - You must not `import *` from a module without a `__all__` (to avoid polluting your globals with their internal stuff, like the stuff they import from elsewhere). The one exception to this is re-exporting stuff (e.g. in `__init__.py`).

benchmark.py

@@ -6,7 +6,7 @@
 from parser.astgen.astgen import AstGen
 from parser.common.tree_print import tformat
 from parser.cst.nodes import ProgramNode
-from parser.cst.treegen import TreeGen
+from parser.cst.cstgen import CstGen
 from parser.lexer import Tokenizer, format_tokens
 from util import readfile
 
@@ -29,7 +29,7 @@ def get(self):
 
 class BenchOnce:
     _tokenizer: Tokenizer
-    _cstgen: TreeGen
+    _cstgen: CstGen
     _cst: ProgramNode
     _ast: AstProgramNode
 
@@ -79,7 +79,7 @@ def do_token_fmt(self):
 
     def do_cst(self):
         with _Timer() as t:
-            self._cstgen = TreeGen(self._tokenizer)
+            self._cstgen = CstGen(self._tokenizer)
             self._cst = self._cstgen.parse()
         self._add_line(1.0, 'CST', t.get())
 

fuzz.py

@@ -2,7 +2,7 @@
 
 from parser.astgen.astgen import AstGen
 from parser.lexer.tokenizer import Tokenizer
-from parser.cst.treegen import TreeGen
+from parser.cst.cstgen import CstGen
 from parser.common.error import BaseParseError
 
 from pythonfuzz.fuzzer import Fuzzer
@@ -30,7 +30,7 @@ def fuzz(buf):
     try:
         string = buf.decode("ascii")
         try:
-            AstGen(TreeGen(Tokenizer(string))).parse()
+            AstGen(CstGen(Tokenizer(string))).parse()
         except BaseParseError:
             pass
     except UnicodeDecodeError:

parser/astgen/astgen.py

@@ -12,7 +12,7 @@
 from ..cst.base_node import AnyNode, Node, Leaf
 from ..cst.named_node import NamedLeafCls, NamedNodeCls, NamedSizedNodeCls
 from ..cst.nodes import *
-from ..cst.treegen import CstGen
+from ..cst.cstgen import CstGen
 
 # Final syntax lowering/codegen: (???)
 # AST -> blocks, intrinsics, functions -> blocks & intrinsics -> intrinsics

parser/cst/treegen.py parser/cst/cstgen.pyparser/cst/treegen.py renamed to parser/cst/cstgen.py

@@ -27,7 +27,7 @@ class LocatedCstError(BaseLocatedError, CstParseError):
     pass
 
 
-class TreeGen:
+class CstGen:
     def __init__(self, tokenizer: Tokenizer):
         self.tokenizer = tokenizer
         self.src = self.tokenizer.src
@@ -542,9 +542,6 @@ def node_from_children(cls, name_or_type: str | type[AnyNamedNode],
         return klass(region, parent, children)
 
 
-CstGen = TreeGen
-
-
 # operator precedence (most to least binding):
 #  1. () [] {}  (parens)
 #  2. string literal auto-concatenation ("ab" "xy" => "abxy")

test/common/common.py

@@ -11,7 +11,7 @@
 from parser.common.error import BaseParseError
 from parser.common.tree_print import tformat
 from parser.cst.base_node import Leaf, AnyNode, Node
-from parser.cst.treegen import TreeGen, CstParseError
+from parser.cst.cstgen import CstGen, CstParseError
 from parser.lexer import Tokenizer
 from parser.lexer.tokens import Token, OpToken
 from test.common.snapshottest import SnapshotTestCase
@@ -83,44 +83,44 @@ def assertTokensEqual(
             self.assertTokenStreamEquals(t.tokens, expected, check_regions)
 
     def assertValidParseCST(self, src: str):
-        self.assertIsNotNone(TreeGen(Tokenizer(src)).parse())
+        self.assertIsNotNone(CstGen(Tokenizer(src)).parse())
 
     def assertFailsGracefullyCST(self, src: str):
-        t = TreeGen(Tokenizer(src))
+        t = CstGen(Tokenizer(src))
         with self.assertRaises(CstParseError) as ctx:
             t.parse()
         return ctx.exception
 
     def assertNotInternalErrorCST(self, src: str):
         try:
-            TreeGen(Tokenizer(src)).parse()
+            CstGen(Tokenizer(src)).parse()
         except BaseParseError:
             self.assertTrue(True)
         self.assertTrue(True)
 
     @classmethod
     def raiseInternalErrorsOnlyCST(cls, src: str):
         try:
-            TreeGen(Tokenizer(src)).parse()
+            CstGen(Tokenizer(src)).parse()
         except BaseParseError:
             return None
         except Exception:
             raise
         return None
 
     def assertCstMatchesSnapshot(self, src: str):
-        t = TreeGen(Tokenizer(src))
+        t = CstGen(Tokenizer(src))
         self.assertMatchesSnapshot(t.parse())
 
     def assertAstMatchesSnapshot(self, src: str):
-        t = AstGen(TreeGen(Tokenizer(src)))
+        t = AstGen(CstGen(Tokenizer(src)))
         self.assertMatchesSnapshot(t.parse())
 
     def assertValidParseAST(self, src: str):
-        self.assertIsNotNone(AstGen(TreeGen(Tokenizer(src))).parse())
+        self.assertIsNotNone(AstGen(CstGen(Tokenizer(src))).parse())
 
     def assertFailsGracefullyAST(self, src: str):
-        a = AstGen(TreeGen(Tokenizer(src)))
+        a = AstGen(CstGen(Tokenizer(src)))
         with self.assertRaises(LocatedAstError) as ctx:
             a.parse()
         return ctx.exception

test/test_cstgen/test_treegen.py

@@ -1,7 +1,7 @@
 import unittest
 
 from parser.lexer.tokenizer import Tokenizer
-from parser.cst.treegen import TreeGen, LocatedCstError, CstGen
+from parser.cst.cstgen import CstGen, LocatedCstError
 from parser.common import StrRegion
 from test.common import CommonTestCase
 
@@ -24,7 +24,7 @@ def test_fn_call_in_lvalue(self):
 
     def test_empty_sqb_error(self):
         with self.assertRaises(LocatedCstError) as err:
-            TreeGen(Tokenizer('v=a[]+b')).parse()
+            CstGen(Tokenizer('v=a[]+b')).parse()
         exc = err.exception
         self.assertBetweenIncl(3, 4, exc.region.start)
         self.assertEqual(4, exc.region.end - 1)
@@ -121,7 +121,7 @@ def test_else_if_else(self):
 
     def test_else_cond_null(self):
         src = 'if 0==1{exit();  } startup();'
-        n = TreeGen(Tokenizer(src)).parse()
+        n = CstGen(Tokenizer(src)).parse()
         node = n.children[0].children[-1]
         self.assertLessEqual(node.region.start, node.region.end)
         self.assertEqual(StrRegion(17, 18), node.region)

test/test_e2e/test_from_fuzzer.py

@@ -2,7 +2,7 @@
 import unittest
 from pathlib import Path
 
-from parser.cst.treegen import TreeGen
+from parser.cst.cstgen import CstGen
 from parser.lexer.tokenizer import Tokenizer
 from test.common import CommonTestCase, TestCaseUtils
 from util import timeout_decor, timeout_decor_async
@@ -60,7 +60,7 @@ def test_a32460d584fd8a20d1e62007db570eaf41342f248e42c21a33780fd976e45290(self):
     @staticmethod
     @timeout_decor(5, debug=0)
     def inner_ed988ae940f54542ec54fd3c402a009fe2fdb660bd558d76a3612781a5ef6aa2():
-        TreeGen(Tokenizer('a;//')).parse()
+        CstGen(Tokenizer('a;//')).parse()
         return CommonTestCase.raiseInternalErrorsOnlyCST('a<//')
 
     def test_ed988ae940f54542ec54fd3c402a009fe2fdb660bd558d76a3612781a5ef6aa2(self):

test/test_e2e/test_main_examples.py

@@ -1,7 +1,7 @@
 import unittest
 
 from parser.astgen.astgen import AstGen
-from parser.cst.treegen import TreeGen
+from parser.cst.cstgen import CstGen
 from parser.lexer import Tokenizer
 from test.common import CommonTestCase
 from util import readfile
@@ -16,7 +16,7 @@ def _test_main_example_n(self, n: int, do_ast: bool = True):
         src = readfile(f'./main_example_{n}.st')
         tk = Tokenizer(src).tokenize()
         self.assertMatchesSnapshot(tk.tokens, 'tokens')
-        t = TreeGen(tk)
+        t = CstGen(tk)
         self.assertMatchesSnapshot(t.parse(), 'cst')
         if do_ast:
             self.assertMatchesSnapshot(AstGen(t).parse(), 'ast')