diff --git a/test/common/__init__.py b/test/common/__init__.py new file mode 100644 index 0000000..c9c4633 --- /dev/null +++ b/test/common/__init__.py @@ -0,0 +1,3 @@ +from .common import CommonTestCase, TokenStreamFlag +from .snapshottest import SnapshotTestCase, SnapshotsNotFound, CantUpdateSnapshots +from .utils import TestCaseUtils diff --git a/test/common.py b/test/common/common.py similarity index 97% rename from test/common.py rename to test/common/common.py index 896b169..890c369 100644 --- a/test/common.py +++ b/test/common/common.py @@ -14,8 +14,8 @@ from parser.cst.treegen import TreeGen, CstParseError from parser.lexer import Tokenizer from parser.lexer.tokens import Token, OpToken -from test.snapshottest import SnapshotTestCase -from test.utils import TestCaseUtils +from test.common.snapshottest import SnapshotTestCase +from test.common.utils import TestCaseUtils def _strict_boundary_kwargs(): diff --git a/test/snapshottest.py b/test/common/snapshottest.py similarity index 100% rename from test/snapshottest.py rename to test/common/snapshottest.py diff --git a/test/utils.py b/test/common/utils.py similarity index 98% rename from test/utils.py rename to test/common/utils.py index 366eac0..aed26fd 100644 --- a/test/utils.py +++ b/test/common/utils.py @@ -69,7 +69,7 @@ def setProperCwd(self): return self._old_cwd = os.getcwd() dirname = Path(__file__).parent - os.chdir(dirname.parent) + os.chdir(dirname.parent.parent) assert self.isProperCwdSet() self.addCleanup(self.resetCwd) diff --git a/test/.snapshots/test_astgen.txt b/test/test_astgen/.snapshots/test_astgen.txt similarity index 100% rename from test/.snapshots/test_astgen.txt rename to test/test_astgen/.snapshots/test_astgen.txt diff --git a/test/test_integration/__init__.py b/test/test_astgen/__init__.py similarity index 100% rename from test/test_integration/__init__.py rename to test/test_astgen/__init__.py diff --git a/test/test_astgen.py b/test/test_astgen/test_astgen.py similarity index 100% rename from test/test_astgen.py rename to test/test_astgen/test_astgen.py diff --git a/test/test_eval_literal.py b/test/test_astgen/test_eval_literal.py similarity index 100% rename from test/test_eval_literal.py rename to test/test_astgen/test_eval_literal.py diff --git a/test/test_integration/.snapshots/test_treegen.txt b/test/test_cstgen/.snapshots/test_treegen.txt similarity index 83% rename from test/test_integration/.snapshots/test_treegen.txt rename to test/test_cstgen/.snapshots/test_treegen.txt index 05e83cd..a61d13f 100644 --- a/test/test_integration/.snapshots/test_treegen.txt +++ b/test/test_cstgen/.snapshots/test_treegen.txt @@ -1,5 +1,5 @@ -"2","45","93","105","114","125","139","158","185","202","229","246","273","301","319","352","371","447" -"TreeGenTest::test_item_chain:0","TreeGenTest::test_fn_call_in_lvalue:0","TreeGenTest::test_aug_assign:0","TreeGenTest::test__mod_supported:0","TreeGenTest::test_decl_no_value:0","TestFunctionDecl::test_no_params:0","TestFunctionDecl::test_one_param:0","TestFunctionDecl::test_two_param:0","TestBlocks::test_while:0","TestBlocks::test_while:1","TestBlocks::test_repeat:0","TestBlocks::test_repeat:1","TestBlocks::test_else_if_else:0","TestBlocks::test_else_if_else:1","TestBlocks::test_else_if_else:2","TestAutocat::test_autocat:0","TreeGenTest::test_decl_multiple:0","TreeGenTest::test_decl:0" +"2","45","93","105","114","125","139","158","185","202","229","246","273","301","319","352","371","447","506","517","528","534","545","556","565" +"TestItemChain::test_item_chain:0","TestItemChain::test_fn_call_in_lvalue:0","TestSmt::test_aug_assign:0","TestExpr::test_mod_supported:0","TestDecl::test_decl_no_value:0","TestFunctionDecl::test_no_params:0","TestFunctionDecl::test_one_param:0","TestFunctionDecl::test_two_param:0","TestBlocks::test_while:0","TestBlocks::test_while:1","TestBlocks::test_repeat:0","TestBlocks::test_repeat:1","TestBlocks::test_else_if_else:0","TestBlocks::test_else_if_else:1","TestBlocks::test_else_if_else:2","TestExpr::test_autocat:0","TestDecl::test_decl_multiple:0","TestDecl::test_decl:0","TestItemChain::test_getattr__issue_09:after_call","TestItemChain::test_getattr__issue_09:after_paren","TestItemChain::test_getattr__issue_09:after_string","TestItemChain::test_getitem__issue_09:after_call","TestItemChain::test_getitem__issue_09:after_paren","TestItemChain::test_getitem__issue_09:after_string","TestSmt::test_empty_smt__issue_04:0" ProgramNode(StrRegion(0, 43), [ AssignNode(StrRegion(0, 42), [ GetitemNode(StrRegion(0, 21), [ @@ -503,4 +503,76 @@ ProgramNode(StrRegion(0, 86), [ ]) ]) ]) +]) +ProgramNode(StrRegion(0, 15), [ + GetattrNode(StrRegion(0, 14), [ + CallNode(StrRegion(0, 12), [ + IdentNode(StrRegion(0, 2)), + CallArgs(StrRegion(2, 12), [ + IdentNode(StrRegion(3, 11)) + ]) + ]), + AttrNameNode(StrRegion(13, 14)) + ]) +]) +ProgramNode(StrRegion(0, 14), [ + GetattrNode(StrRegion(0, 13), [ + ParenNode(StrRegion(0, 11), [ + AddNode(StrRegion(1, 10), [ + IdentNode(StrRegion(1, 6)), + IdentNode(StrRegion(9, 10)) + ]) + ]), + AttrNameNode(StrRegion(12, 13)) + ]) +]) +ProgramNode(StrRegion(0, 13), [ + GetattrNode(StrRegion(0, 12), [ + StringNode(StrRegion(0, 10)), + AttrNameNode(StrRegion(11, 12)) + ]) +]) +ProgramNode(StrRegion(0, 16), [ + GetitemNode(StrRegion(0, 15), [ + CallNode(StrRegion(0, 12), [ + IdentNode(StrRegion(0, 2)), + CallArgs(StrRegion(2, 12), [ + IdentNode(StrRegion(3, 11)) + ]) + ]), + NumberNode(StrRegion(13, 14)) + ]) +]) +ProgramNode(StrRegion(0, 15), [ + GetitemNode(StrRegion(0, 14), [ + ParenNode(StrRegion(0, 11), [ + AddNode(StrRegion(1, 10), [ + IdentNode(StrRegion(1, 6)), + IdentNode(StrRegion(9, 10)) + ]) + ]), + NumberNode(StrRegion(12, 13)) + ]) +]) +ProgramNode(StrRegion(0, 24), [ + GetitemNode(StrRegion(0, 23), [ + StringNode(StrRegion(0, 10)), + ConcatNode(StrRegion(11, 22), [ + StringNode(StrRegion(11, 17)), + NumberNode(StrRegion(21, 22)) + ]) + ]) +]) +ProgramNode(StrRegion(0, 9), [ + DeclNode(StrRegion(0, 7), [ + DeclScope_Let(StrRegion(0, 3)), + DeclType_Variable(StrRegion(3, 3)), + DeclItemsList(StrRegion(4, 7), [ + DeclItemNode(StrRegion(4, 7), [ + IdentNode(StrRegion(4, 5)), + NumberNode(StrRegion(6, 7)) + ]) + ]) + ]), + NopNode(StrRegion(8, 9)) ]) \ No newline at end of file diff --git a/test/test_cstgen/__init__.py b/test/test_cstgen/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_integration/test_treegen.py b/test/test_cstgen/test_eof_handling.py similarity index 65% rename from test/test_integration/test_treegen.py rename to test/test_cstgen/test_eof_handling.py index 6a45fab..4f3526f 100644 --- a/test/test_integration/test_treegen.py +++ b/test/test_cstgen/test_eof_handling.py @@ -1,100 +1,7 @@ -import unittest - -from parser.lexer.tokenizer import Tokenizer from parser.operators import BINARY_OPS -from parser.cst.treegen import TreeGen, LocatedCstError -from parser.common import StrRegion from test.common import CommonTestCase -class TestAutocat(CommonTestCase): - def test_autocat(self): - self.assertCstMatchesSnapshot('"abc" "="\n "1" .. a .. "b".d();') - - -class TreeGenTest(CommonTestCase): - def test_item_chain(self): - self.assertCstMatchesSnapshot('a[7].b.0.fn["c" .. 2] = fn(9).k[7 + r](3,);') - - def test_fn_call_in_lvalue(self): - self.assertCstMatchesSnapshot('a(7).b.0.fn()["c" .. 2] = fn(9).k[7 + r](3,);') - - def test_aug_assign(self): - self.assertCstMatchesSnapshot('a[1] += a.2;') - - def test__mod_supported(self): - self.assertCstMatchesSnapshot('c=a%b;') - - def test_decl_no_value(self): - self.assertCstMatchesSnapshot('let b;') - - def test_decl_multiple(self): - self.assertCstMatchesSnapshot( - 'let a, b=9, c,d=w.1[2],e="w",f,g;\n' - 'global z,y=-11, x , w="a".lower() ,v=o , u, t;') - - def test_decl(self): - self.assertCstMatchesSnapshot('let a,b=1+1,c;\n' - 'global d = "STRING", e;\n' - 'let[] local_list=list(), other;\n' - 'global[] STACK;') - - def test_decl_error(self): - err = self.assertFailsGracefullyCST('let[4] = 9;') - self.assertEqual(StrRegion(4, 5), err.region) - self.assertContains(err.msg.lower(), "expected ']' after '['") - - err = self.assertFailsGracefullyCST('let[') - self.assertEqual(StrRegion(4, 4), err.region) - self.assertContains(err.msg.lower(), "expected ']' after '['") - - -class TestBlocks(CommonTestCase): - def test_while(self): - self.assertCstMatchesSnapshot('while a || !b && c >= 6 {}') - self.assertCstMatchesSnapshot('while!(7%8){(7.7).abc(6,7,8);}') - - def test_repeat(self): - self.assertCstMatchesSnapshot('repeat a || !b && c >= 6 {}') - self.assertCstMatchesSnapshot('repeat!(7%8){(7.7).abc(6,7,8);}') - - def test_else_if_else(self): - self.assertCstMatchesSnapshot('if(1){}else if(a||!b&&c!=6){}') - self.assertCstMatchesSnapshot('if(1){}else{a();}') - self.assertCstMatchesSnapshot('if(1){}else if 9{a();}else{b(a, a());}') - - def test_else_cond_null(self): - src = 'if 0==1{exit(); } startup();' - n = TreeGen(Tokenizer(src)).parse() - node = n.children[0].children[-1] - self.assertLessEqual(node.region.start, node.region.end) - self.assertEqual(StrRegion(17, 18), node.region) - - -class TestFunctionDecl(CommonTestCase): - def setUp(self) -> None: - super().setUp() - self.setProperCwd() - - def test_no_params(self): - self.assertCstMatchesSnapshot('def a() { alert("Called"); }') - - def test_one_param(self): - self.assertCstMatchesSnapshot('def a(number val){print(val);}') - - def test_two_param(self): - self.assertCstMatchesSnapshot('def a(number a, string b){RESULT=a.."="..b;}') - - -class TestTreeGenErrors(CommonTestCase): - def test_empty_sqb_error(self): - with self.assertRaises(LocatedCstError) as err: - TreeGen(Tokenizer('v=a[]+b')).parse() - exc = err.exception - self.assertBetweenIncl(3, 4, exc.region.start) - self.assertEqual(4, exc.region.end - 1) - - class TreeGenEofTest(CommonTestCase): def test__at_expr_end(self): self.assertFailsGracefullyCST('a.b()') @@ -302,7 +209,3 @@ def test__bin_op__success(self): for op in BINARY_OPS: with self.subTest(op=op): self.assertValidParseCST(f'a{op}b;') - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_node.py b/test/test_cstgen/test_node.py similarity index 100% rename from test/test_node.py rename to test/test_cstgen/test_node.py diff --git a/test/test_cstgen/test_treegen.py b/test/test_cstgen/test_treegen.py new file mode 100644 index 0000000..31548d0 --- /dev/null +++ b/test/test_cstgen/test_treegen.py @@ -0,0 +1,147 @@ +import unittest + +from parser.lexer.tokenizer import Tokenizer +from parser.cst.treegen import TreeGen, LocatedCstError, CstGen +from parser.common import StrRegion +from test.common import CommonTestCase + + +# region ---- ---- +class TestExpr(CommonTestCase): + def test_autocat(self): + self.assertCstMatchesSnapshot('"abc" "="\n "1" .. a .. "b".d();') + + def test_mod_supported(self): + self.assertCstMatchesSnapshot('c=a%b;') + + +class TestItemChain(CommonTestCase): + def test_item_chain(self): + self.assertCstMatchesSnapshot('a[7].b.0.fn["c" .. 2] = fn(9).k[7 + r](3,);') + + def test_fn_call_in_lvalue(self): + self.assertCstMatchesSnapshot('a(7).b.0.fn()["c" .. 2] = fn(9).k[7 + r](3,);') + + def test_empty_sqb_error(self): + with self.assertRaises(LocatedCstError) as err: + TreeGen(Tokenizer('v=a[]+b')).parse() + exc = err.exception + self.assertBetweenIncl(3, 4, exc.region.start) + self.assertEqual(4, exc.region.end - 1) + + def test_getattr__issue_09(self): + t = Tokenizer('fn(call_arg).a;').tokenize() + node = CstGen(t).parse() + self.assertMatchesSnapshot(node, 'after_call') + t = Tokenizer('(paren + x).b;').tokenize() + node = CstGen(t).parse() + self.assertMatchesSnapshot(node, 'after_paren') + t = Tokenizer('"a string".b;').tokenize() + node = CstGen(t).parse() + self.assertMatchesSnapshot(node, 'after_string') + + def test_getitem__issue_09(self): + t = Tokenizer('fn(call_arg)[1];').tokenize() + node = CstGen(t).parse() + self.assertMatchesSnapshot(node, 'after_call') + t = Tokenizer('(paren + x)[2];').tokenize() + node = CstGen(t).parse() + self.assertMatchesSnapshot(node, 'after_paren') + t = Tokenizer('"a string"["key_" .. 3];').tokenize() + node = CstGen(t).parse() + self.assertMatchesSnapshot(node, 'after_string') +# endregion + + +# region ---- ---- +class TestSmt(CommonTestCase): + def test_aug_assign(self): + self.assertCstMatchesSnapshot('a[1] += a.2;') + + def test_empty_smt__issue_04(self): + t = Tokenizer('let a=9;;').tokenize() + node = CstGen(t).parse() + self.assertMatchesSnapshot(node) + + +class TestDecl(CommonTestCase): + def test_empty_assign_source_error(self): + t = Tokenizer('let a= ;').tokenize() + with self.assertRaises(LocatedCstError) as err: + CstGen(t).parse() + self.assertBetweenIncl(5, 7, err.exception.region.start) + self.assertBetweenIncl(7, 8, err.exception.region.end) + self.assertContains(str(err.exception), "semi") + + def test_decl_no_value(self): + self.assertCstMatchesSnapshot('let b;') + + def test_decl_multiple(self): + self.assertCstMatchesSnapshot( + 'let a, b=9, c,d=w.1[2],e="w",f,g;\n' + 'global z,y=-11, x , w="a".lower() ,v=o , u, t;') + + def test_decl(self): + self.assertCstMatchesSnapshot('let a,b=1+1,c;\n' + 'global d = "STRING", e;\n' + 'let[] local_list=list(), other;\n' + 'global[] STACK;') + + def test_decl_error(self): + err = self.assertFailsGracefullyCST('let[4] = 9;') + self.assertEqual(StrRegion(4, 5), err.region) + self.assertContains(err.msg.lower(), "expected ']' after '['") + + err = self.assertFailsGracefullyCST('let[') + self.assertEqual(StrRegion(4, 4), err.region) + self.assertContains(err.msg.lower(), "expected ']' after '['") + + +class TestBlocks(CommonTestCase): + def test_while(self): + self.assertCstMatchesSnapshot('while a || !b && c >= 6 {}') + self.assertCstMatchesSnapshot('while!(7%8){(7.7).abc(6,7,8);}') + + def test_repeat(self): + self.assertCstMatchesSnapshot('repeat a || !b && c >= 6 {}') + self.assertCstMatchesSnapshot('repeat!(7%8){(7.7).abc(6,7,8);}') + + def test_empty_condition_error(self): + t = Tokenizer('if {x();}').tokenize() + with self.assertRaises(LocatedCstError) as err: + CstGen(t).parse() + self.assertBetweenIncl(0, 3, err.exception.region.start) + self.assertBetweenIncl(2, 4, err.exception.region.end) + self.assertContains(str(err.exception), "brace") + + def test_else_if_else(self): + self.assertCstMatchesSnapshot('if(1){}else if(a||!b&&c!=6){}') + self.assertCstMatchesSnapshot('if(1){}else{a();}') + self.assertCstMatchesSnapshot('if(1){}else if 9{a();}else{b(a, a());}') + + def test_else_cond_null(self): + src = 'if 0==1{exit(); } startup();' + n = TreeGen(Tokenizer(src)).parse() + node = n.children[0].children[-1] + self.assertLessEqual(node.region.start, node.region.end) + self.assertEqual(StrRegion(17, 18), node.region) + + +class TestFunctionDecl(CommonTestCase): + def setUp(self) -> None: + super().setUp() + self.setProperCwd() + + def test_no_params(self): + self.assertCstMatchesSnapshot('def a() { alert("Called"); }') + + def test_one_param(self): + self.assertCstMatchesSnapshot('def a(number val){print(val);}') + + def test_two_param(self): + self.assertCstMatchesSnapshot('def a(number a, string b){RESULT=a.."="..b;}') +# endregion + + +if __name__ == '__main__': + unittest.main() diff --git a/test/.snapshots/test_main_examples.txt b/test/test_e2e/.snapshots/test_main_examples.txt similarity index 100% rename from test/.snapshots/test_main_examples.txt rename to test/test_e2e/.snapshots/test_main_examples.txt diff --git a/test/test_e2e/__init__.py b/test/test_e2e/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_from_fuzzer.py b/test/test_e2e/test_from_fuzzer.py similarity index 97% rename from test/test_from_fuzzer.py rename to test/test_e2e/test_from_fuzzer.py index fcc1037..569e790 100644 --- a/test/test_from_fuzzer.py +++ b/test/test_e2e/test_from_fuzzer.py @@ -2,11 +2,9 @@ import unittest from pathlib import Path -from parser.lexer.tokenizer import Tokenizer from parser.cst.treegen import TreeGen -from test.common import CommonTestCase - -from test.utils import TestCaseUtils +from parser.lexer.tokenizer import Tokenizer +from test.common import CommonTestCase, TestCaseUtils from util import timeout_decor, timeout_decor_async diff --git a/test/test_main_examples.py b/test/test_e2e/test_main_examples.py similarity index 100% rename from test/test_main_examples.py rename to test/test_e2e/test_main_examples.py diff --git a/test/test_integration/.snapshots/test_empty_expr.txt b/test/test_integration/.snapshots/test_empty_expr.txt deleted file mode 100644 index 652c8c2..0000000 --- a/test/test_integration/.snapshots/test_empty_expr.txt +++ /dev/null @@ -1,15 +0,0 @@ -"2" -"MyTestCase::test_empty_expr_issue_04:0" -ProgramNode(StrRegion(0, 9), [ - DeclNode(StrRegion(0, 7), [ - DeclScope_Let(StrRegion(0, 3)), - DeclType_Variable(StrRegion(3, 3)), - DeclItemsList(StrRegion(4, 7), [ - DeclItemNode(StrRegion(4, 7), [ - IdentNode(StrRegion(4, 5)), - NumberNode(StrRegion(6, 7)) - ]) - ]) - ]), - NopNode(StrRegion(8, 9)) -]) \ No newline at end of file diff --git a/test/test_integration/.snapshots/test_parse_expr_pass_0.txt b/test/test_integration/.snapshots/test_parse_expr_pass_0.txt deleted file mode 100644 index a88baf0..0000000 --- a/test/test_integration/.snapshots/test_parse_expr_pass_0.txt +++ /dev/null @@ -1,61 +0,0 @@ -"2","13","24","35","41","52" -"MyTestCase::test_issue_09__dot:after_call","MyTestCase::test_issue_09__sqb:after_call","MyTestCase::test_issue_09__dot:after_paren","MyTestCase::test_issue_09__dot:after_string","MyTestCase::test_issue_09__sqb:after_paren","MyTestCase::test_issue_09__sqb:after_string" -ProgramNode(StrRegion(0, 15), [ - GetattrNode(StrRegion(0, 14), [ - CallNode(StrRegion(0, 12), [ - IdentNode(StrRegion(0, 2)), - CallArgs(StrRegion(2, 12), [ - IdentNode(StrRegion(3, 11)) - ]) - ]), - AttrNameNode(StrRegion(13, 14)) - ]) -]) -ProgramNode(StrRegion(0, 16), [ - GetitemNode(StrRegion(0, 15), [ - CallNode(StrRegion(0, 12), [ - IdentNode(StrRegion(0, 2)), - CallArgs(StrRegion(2, 12), [ - IdentNode(StrRegion(3, 11)) - ]) - ]), - NumberNode(StrRegion(13, 14)) - ]) -]) -ProgramNode(StrRegion(0, 14), [ - GetattrNode(StrRegion(0, 13), [ - ParenNode(StrRegion(0, 11), [ - AddNode(StrRegion(1, 10), [ - IdentNode(StrRegion(1, 6)), - IdentNode(StrRegion(9, 10)) - ]) - ]), - AttrNameNode(StrRegion(12, 13)) - ]) -]) -ProgramNode(StrRegion(0, 13), [ - GetattrNode(StrRegion(0, 12), [ - StringNode(StrRegion(0, 10)), - AttrNameNode(StrRegion(11, 12)) - ]) -]) -ProgramNode(StrRegion(0, 15), [ - GetitemNode(StrRegion(0, 14), [ - ParenNode(StrRegion(0, 11), [ - AddNode(StrRegion(1, 10), [ - IdentNode(StrRegion(1, 6)), - IdentNode(StrRegion(9, 10)) - ]) - ]), - NumberNode(StrRegion(12, 13)) - ]) -]) -ProgramNode(StrRegion(0, 24), [ - GetitemNode(StrRegion(0, 23), [ - StringNode(StrRegion(0, 10)), - ConcatNode(StrRegion(11, 22), [ - StringNode(StrRegion(11, 17)), - NumberNode(StrRegion(21, 22)) - ]) - ]) -]) \ No newline at end of file diff --git a/test/test_integration/test_empty_expr.py b/test/test_integration/test_empty_expr.py deleted file mode 100644 index 8b29a8f..0000000 --- a/test/test_integration/test_empty_expr.py +++ /dev/null @@ -1,33 +0,0 @@ -import unittest - -from test.common import CommonTestCase -from parser.lexer import Tokenizer -from parser.cst.treegen import CstGen, LocatedCstError - - -class MyTestCase(CommonTestCase): - - def test_error_empty_assign_source(self): - t = Tokenizer('let a= ;').tokenize() - with self.assertRaises(LocatedCstError) as err: - CstGen(t).parse() - self.assertBetweenIncl(5, 7, err.exception.region.start) - self.assertBetweenIncl(7, 8, err.exception.region.end) - self.assertContains(str(err.exception), "semi") - - def test_error_empty_condition(self): - t = Tokenizer('if {x();}').tokenize() - with self.assertRaises(LocatedCstError) as err: - CstGen(t).parse() - self.assertBetweenIncl(0, 3, err.exception.region.start) - self.assertBetweenIncl(2, 4, err.exception.region.end) - self.assertContains(str(err.exception), "brace") - - def test_empty_expr_issue_04(self): - t = Tokenizer('let a=9;;').tokenize() - node = CstGen(t).parse() - self.assertMatchesSnapshot(node) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_integration/test_parse_expr_pass_0.py b/test/test_integration/test_parse_expr_pass_0.py deleted file mode 100644 index 21c1594..0000000 --- a/test/test_integration/test_parse_expr_pass_0.py +++ /dev/null @@ -1,33 +0,0 @@ -import unittest - -from parser.cst.treegen import CstGen -from parser.lexer import Tokenizer -from test.common import CommonTestCase - - -class MyTestCase(CommonTestCase): - def test_issue_09__dot(self): - t = Tokenizer('fn(call_arg).a;').tokenize() - node = CstGen(t).parse() - self.assertMatchesSnapshot(node, 'after_call') - t = Tokenizer('(paren + x).b;').tokenize() - node = CstGen(t).parse() - self.assertMatchesSnapshot(node, 'after_paren') - t = Tokenizer('"a string".b;').tokenize() - node = CstGen(t).parse() - self.assertMatchesSnapshot(node, 'after_string') - - def test_issue_09__sqb(self): - t = Tokenizer('fn(call_arg)[1];').tokenize() - node = CstGen(t).parse() - self.assertMatchesSnapshot(node, 'after_call') - t = Tokenizer('(paren + x)[2];').tokenize() - node = CstGen(t).parse() - self.assertMatchesSnapshot(node, 'after_paren') - t = Tokenizer('"a string"["key_" .. 3];').tokenize() - node = CstGen(t).parse() - self.assertMatchesSnapshot(node, 'after_string') - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_integration/test_ws_at_end_issue_001.py b/test/test_integration/test_ws_at_end_issue_001.py deleted file mode 100644 index 6e3230b..0000000 --- a/test/test_integration/test_ws_at_end_issue_001.py +++ /dev/null @@ -1,25 +0,0 @@ -import unittest - -from parser.lexer import Tokenizer -from parser.tokens import * -from test.common import CommonTestCase, TokenStreamFlag - - -class MyTestCase(CommonTestCase): - def test_ws_at_end(self): - t = Tokenizer('let a =1; \n').tokenize() - self.assertTokensEqual(t, [ - IdentNameToken(), - WhitespaceToken(), - IdentNameToken(), - WhitespaceToken(), - OpToken(op_str='='), - NumberToken(), - SemicolonToken(), - WhitespaceToken(), - EofToken() - ], TokenStreamFlag.FULL, check_regions=False) - - -if __name__ == '__main__': - unittest.main() diff --git a/test/test_tokenizer/__init__.py b/test/test_tokenizer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/test/test_tokenizer.py b/test/test_tokenizer/test_tokenizer.py similarity index 84% rename from test/test_tokenizer.py rename to test/test_tokenizer/test_tokenizer.py index 29c2e17..bd48eb6 100644 --- a/test/test_tokenizer.py +++ b/test/test_tokenizer/test_tokenizer.py @@ -1,13 +1,14 @@ import unittest from parser.lexer import Tokenizer -from parser.lexer.tokens import WhitespaceToken, StringToken, EofToken, NumberToken +from parser.lexer.tokens import ( + WhitespaceToken, StringToken, EofToken, NumberToken, SemicolonToken) from parser.common import StrRegion from parser.tokens import IdentNameToken, DotToken, AttrNameToken, OpToken from test.common import CommonTestCase, TokenStreamFlag -class MyTestCase(CommonTestCase): +class TestInternalFuncs(CommonTestCase): def test__t_ident_name__at_end(self): t = Tokenizer('abc') end = t._t_ident_name(0) @@ -63,6 +64,8 @@ def test__t_number(self): self.assertTokensEqual(t, [NumberToken(StrRegion(0, 2))]) self.assertEqual(end, 2) + +class TestFullTokenizer(CommonTestCase): def test_mod_supported(self): t = Tokenizer('a+b%2') t.tokenize() @@ -93,6 +96,20 @@ def test_tokenize_concat_works(self): EofToken(StrRegion(9, 9)), ], TokenStreamFlag.CONTENT) + def test_ws_at_end(self): + t = Tokenizer('let a =1; \n').tokenize() + self.assertTokensEqual(t, [ + IdentNameToken(), + WhitespaceToken(), + IdentNameToken(), + WhitespaceToken(), + OpToken(op_str='='), + NumberToken(), + SemicolonToken(), + WhitespaceToken(), + EofToken() + ], TokenStreamFlag.FULL, check_regions=False) + if __name__ == '__main__': unittest.main()