diff --git a/README.md b/README.md index 3adf0cc..3b79c1a 100644 --- a/README.md +++ b/README.md @@ -41,7 +41,7 @@ - Importing from outside this project: no (we don't know what extra stuff other projects put in their module namespace that might break our code). - Importing internally within the project: - Useful when you need *everything* from a module that has many *similar* classes - - (e.g. importing all the nodes (`parser/cst/nodes.py`) in the CST generation code (`parser/cst/treegen.py`) is good). + - (e.g. importing all the nodes (`parser/cst/nodes.py`) in the CST generation code (`parser/cst/cstgen.py`) is good). - The threshold for 'many' here is around 10-15 classes. - Otherwise (if the classes/functions are unrelated (e.g. in `utils.py`), or you don't need all/most of them, or the module has few classes), just list them out. - You must not `import *` from a module without a `__all__` (to avoid polluting your globals with their internal stuff, like the stuff they import from elsewhere). The one exception to this is re-exporting stuff (e.g. in `__init__.py`). diff --git a/benchmark.py b/benchmark.py index 0cfec88..9c34707 100644 --- a/benchmark.py +++ b/benchmark.py @@ -6,7 +6,7 @@ from parser.astgen.astgen import AstGen from parser.common.tree_print import tformat from parser.cst.nodes import ProgramNode -from parser.cst.treegen import TreeGen +from parser.cst.cstgen import CstGen from parser.lexer import Tokenizer, format_tokens from util import readfile @@ -29,7 +29,7 @@ def get(self): class BenchOnce: _tokenizer: Tokenizer - _cstgen: TreeGen + _cstgen: CstGen _cst: ProgramNode _ast: AstProgramNode @@ -79,7 +79,7 @@ def do_token_fmt(self): def do_cst(self): with _Timer() as t: - self._cstgen = TreeGen(self._tokenizer) + self._cstgen = CstGen(self._tokenizer) self._cst = self._cstgen.parse() self._add_line(1.0, 'CST', t.get()) diff --git a/fuzz.py b/fuzz.py index a06b13e..cb510b8 100644 --- a/fuzz.py +++ b/fuzz.py @@ -2,7 +2,7 @@ from parser.astgen.astgen import AstGen from parser.lexer.tokenizer import Tokenizer -from parser.cst.treegen import TreeGen +from parser.cst.cstgen import CstGen from parser.common.error import BaseParseError from pythonfuzz.fuzzer import Fuzzer @@ -30,7 +30,7 @@ def fuzz(buf): try: string = buf.decode("ascii") try: - AstGen(TreeGen(Tokenizer(string))).parse() + AstGen(CstGen(Tokenizer(string))).parse() except BaseParseError: pass except UnicodeDecodeError: diff --git a/parser/astgen/astgen.py b/parser/astgen/astgen.py index 11e7d86..bb7a36c 100644 --- a/parser/astgen/astgen.py +++ b/parser/astgen/astgen.py @@ -12,7 +12,7 @@ from ..cst.base_node import AnyNode, Node, Leaf from ..cst.named_node import NamedLeafCls, NamedNodeCls, NamedSizedNodeCls from ..cst.nodes import * -from ..cst.treegen import CstGen +from ..cst.cstgen import CstGen # Final syntax lowering/codegen: (???) # AST -> blocks, intrinsics, functions -> blocks & intrinsics -> intrinsics diff --git a/parser/cst/treegen.py b/parser/cst/cstgen.py similarity index 99% rename from parser/cst/treegen.py rename to parser/cst/cstgen.py index 4afca79..50c5283 100644 --- a/parser/cst/treegen.py +++ b/parser/cst/cstgen.py @@ -27,7 +27,7 @@ class LocatedCstError(BaseLocatedError, CstParseError): pass -class TreeGen: +class CstGen: def __init__(self, tokenizer: Tokenizer): self.tokenizer = tokenizer self.src = self.tokenizer.src @@ -542,9 +542,6 @@ def node_from_children(cls, name_or_type: str | type[AnyNamedNode], return klass(region, parent, children) -CstGen = TreeGen - - # operator precedence (most to least binding): # 1. () [] {} (parens) # 2. string literal auto-concatenation ("ab" "xy" => "abxy") diff --git a/test/common/common.py b/test/common/common.py index 890c369..8948baa 100644 --- a/test/common/common.py +++ b/test/common/common.py @@ -11,7 +11,7 @@ from parser.common.error import BaseParseError from parser.common.tree_print import tformat from parser.cst.base_node import Leaf, AnyNode, Node -from parser.cst.treegen import TreeGen, CstParseError +from parser.cst.cstgen import CstGen, CstParseError from parser.lexer import Tokenizer from parser.lexer.tokens import Token, OpToken from test.common.snapshottest import SnapshotTestCase @@ -83,17 +83,17 @@ def assertTokensEqual( self.assertTokenStreamEquals(t.tokens, expected, check_regions) def assertValidParseCST(self, src: str): - self.assertIsNotNone(TreeGen(Tokenizer(src)).parse()) + self.assertIsNotNone(CstGen(Tokenizer(src)).parse()) def assertFailsGracefullyCST(self, src: str): - t = TreeGen(Tokenizer(src)) + t = CstGen(Tokenizer(src)) with self.assertRaises(CstParseError) as ctx: t.parse() return ctx.exception def assertNotInternalErrorCST(self, src: str): try: - TreeGen(Tokenizer(src)).parse() + CstGen(Tokenizer(src)).parse() except BaseParseError: self.assertTrue(True) self.assertTrue(True) @@ -101,7 +101,7 @@ def assertNotInternalErrorCST(self, src: str): @classmethod def raiseInternalErrorsOnlyCST(cls, src: str): try: - TreeGen(Tokenizer(src)).parse() + CstGen(Tokenizer(src)).parse() except BaseParseError: return None except Exception: @@ -109,18 +109,18 @@ def raiseInternalErrorsOnlyCST(cls, src: str): return None def assertCstMatchesSnapshot(self, src: str): - t = TreeGen(Tokenizer(src)) + t = CstGen(Tokenizer(src)) self.assertMatchesSnapshot(t.parse()) def assertAstMatchesSnapshot(self, src: str): - t = AstGen(TreeGen(Tokenizer(src))) + t = AstGen(CstGen(Tokenizer(src))) self.assertMatchesSnapshot(t.parse()) def assertValidParseAST(self, src: str): - self.assertIsNotNone(AstGen(TreeGen(Tokenizer(src))).parse()) + self.assertIsNotNone(AstGen(CstGen(Tokenizer(src))).parse()) def assertFailsGracefullyAST(self, src: str): - a = AstGen(TreeGen(Tokenizer(src))) + a = AstGen(CstGen(Tokenizer(src))) with self.assertRaises(LocatedAstError) as ctx: a.parse() return ctx.exception diff --git a/test/test_cstgen/.snapshots/test_treegen.txt b/test/test_cstgen/.snapshots/test_cstgen.txt similarity index 100% rename from test/test_cstgen/.snapshots/test_treegen.txt rename to test/test_cstgen/.snapshots/test_cstgen.txt diff --git a/test/test_cstgen/test_treegen.py b/test/test_cstgen/test_cstgen.py similarity index 97% rename from test/test_cstgen/test_treegen.py rename to test/test_cstgen/test_cstgen.py index 31548d0..efeb8c6 100644 --- a/test/test_cstgen/test_treegen.py +++ b/test/test_cstgen/test_cstgen.py @@ -1,7 +1,7 @@ import unittest from parser.lexer.tokenizer import Tokenizer -from parser.cst.treegen import TreeGen, LocatedCstError, CstGen +from parser.cst.cstgen import CstGen, LocatedCstError from parser.common import StrRegion from test.common import CommonTestCase @@ -24,7 +24,7 @@ def test_fn_call_in_lvalue(self): def test_empty_sqb_error(self): with self.assertRaises(LocatedCstError) as err: - TreeGen(Tokenizer('v=a[]+b')).parse() + CstGen(Tokenizer('v=a[]+b')).parse() exc = err.exception self.assertBetweenIncl(3, 4, exc.region.start) self.assertEqual(4, exc.region.end - 1) @@ -121,7 +121,7 @@ def test_else_if_else(self): def test_else_cond_null(self): src = 'if 0==1{exit(); } startup();' - n = TreeGen(Tokenizer(src)).parse() + n = CstGen(Tokenizer(src)).parse() node = n.children[0].children[-1] self.assertLessEqual(node.region.start, node.region.end) self.assertEqual(StrRegion(17, 18), node.region) diff --git a/test/test_cstgen/test_eof_handling.py b/test/test_cstgen/test_eof_handling.py index 4f3526f..ebd893f 100644 --- a/test/test_cstgen/test_eof_handling.py +++ b/test/test_cstgen/test_eof_handling.py @@ -2,7 +2,7 @@ from test.common import CommonTestCase -class TreeGenEofTest(CommonTestCase): +class TestCstGenEof(CommonTestCase): def test__at_expr_end(self): self.assertFailsGracefullyCST('a.b()') diff --git a/test/test_e2e/test_from_fuzzer.py b/test/test_e2e/test_from_fuzzer.py index 569e790..a5d6dd3 100644 --- a/test/test_e2e/test_from_fuzzer.py +++ b/test/test_e2e/test_from_fuzzer.py @@ -2,7 +2,7 @@ import unittest from pathlib import Path -from parser.cst.treegen import TreeGen +from parser.cst.cstgen import CstGen from parser.lexer.tokenizer import Tokenizer from test.common import CommonTestCase, TestCaseUtils from util import timeout_decor, timeout_decor_async @@ -60,7 +60,7 @@ def test_a32460d584fd8a20d1e62007db570eaf41342f248e42c21a33780fd976e45290(self): @staticmethod @timeout_decor(5, debug=0) def inner_ed988ae940f54542ec54fd3c402a009fe2fdb660bd558d76a3612781a5ef6aa2(): - TreeGen(Tokenizer('a;//')).parse() + CstGen(Tokenizer('a;//')).parse() return CommonTestCase.raiseInternalErrorsOnlyCST('a