Merge pull request #44 from MarcellPerger1/start-ast

Add AST generation

Author: MarcellPerger1

.coveragerc

@@ -0,0 +1,5 @@
+[report]
+exclude_also =
+    if __name__ == ['"]__main__['"]:
+    if TYPE_CHECKING:
+    assert 0\b

.github/workflows/fuzzer.yml

@@ -5,16 +5,34 @@ on:
         description: Number of iterations
         type: number
         default: 250000
+      do_shard:
+        description: Run it on 4 separate machines (`n` on each)?
+        type: boolean
+        default: false
 
 
 jobs:
-  test:
+  fuzzer:
     runs-on: ubuntu-latest
     strategy:
       matrix:
         py_version: 
           - "3.10"
           - "3.11"
+          - "3.12"
+        do_shard:
+          - ${{ inputs.do_shard }}
+        shard_index: [0, 1, 2, 3]
+        exclude:
+          - do_shard: false
+        include:
+          - do_shard: false
+            py_version: "3.10"
+          - do_shard: false
+            py_version: "3.11"
+          - do_shard: false
+            py_version: "3.12"
+    name: ${{ inputs.do_shard && format('Run fuzzer (Python {0}, shard {1})', matrix.py_version, matrix.shard_index) || format('Run fuzzer (Python {0})', matrix.py_version) }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

.github/workflows/run_tests.yml

@@ -15,6 +15,7 @@ jobs:
         py_version: 
           - "3.10"
           - "3.11"
+          - "3.12"
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4

.idea/inspectionProfiles/project_inspections.xml

@@ -1,5 +1,6 @@
 import time
 
+from parser.astgen.astgen import AstGen
 from parser.lexer.tokenizer import Tokenizer
 from parser.cst.treegen import TreeGen
 from parser.common.error import BaseParseError
@@ -29,7 +30,7 @@ def fuzz(buf):
     try:
         string = buf.decode("ascii")
         try:
-            TreeGen(Tokenizer(string)).parse()
+            AstGen(TreeGen(Tokenizer(string))).parse()
         except BaseParseError:
             pass
     except UnicodeDecodeError:
@@ -39,11 +40,12 @@ def fuzz(buf):
 if __name__ == '__main__':
     import argparse
     ap = argparse.ArgumentParser("fuzz.py", description="Runs a fuzzer for n iterations")
+    # Use type=float as gh mobile cannot specify integers as workflow args
     ap.add_argument('-n', '--iterations', default=-1,
-                    type=int, help="Number of iterations to run pythonfuzz for")
+                    type=float, help="Number of iterations to run pythonfuzz for")
     ap.add_argument('-i', '--infinite',
                     action='store_const', const=-1, dest='iterations')
     args = ap.parse_args()
 
-    fuzzer = Fuzzer(fuzz, dirs=['./pythonfuzz_corpus'], timeout=30, runs=args.iterations)
+    fuzzer = Fuzzer(fuzz, dirs=['./pythonfuzz_corpus'], timeout=30, runs=int(args.iterations))
     fuzzer.start()

fuzz.py

@@ -1,6 +1,7 @@
 import cProfile
 import time
 
+from parser.astgen.astgen import AstGen
 from util import readfile
 from parser.cst.treegen import TreeGen
 from parser.lexer import Tokenizer, print_tokens
@@ -14,7 +15,20 @@ def make_tree(src: str):
 PROFILER = True
 
 
-def run(src: str, idx: int = -1):
+def run(src: str, idx: int = -1, do_ast=True):
+    node = ast_node = None
+    ta1 = tp1 = ta0 = 0.0  # will be overwritten
+
+    def doit_trees():
+        nonlocal node, tp1, ta0, ast_node, ta1
+        treegen = TreeGen(tn)
+        node = treegen.parse()
+        tp1 = time.perf_counter()
+        if do_ast:
+            ta0 = time.perf_counter()
+            ast_node = AstGen(treegen).parse()
+            ta1 = time.perf_counter()
+
     tn0 = time.perf_counter()
     tn = Tokenizer(src).tokenize()
     tn1 = time.perf_counter()
@@ -25,24 +39,29 @@ def run(src: str, idx: int = -1):
     tp0 = time.perf_counter()
     if PROFILER:
         with cProfile.Profile() as p:
-            node = TreeGen(tn).parse()
-        tp1 = time.perf_counter()
+            doit_trees()
         p.dump_stats(f'perf_dump_{idx}.prof')
     else:
-        node = TreeGen(tn).parse()
-        tp1 = time.perf_counter()
+        doit_trees()
     print('CST:')
     tpr_cst0 = time.perf_counter()
     tprint(node)
     tpr_cst1 = time.perf_counter()
+    tpr_ast0 = tpr_ast1 = time.perf_counter()
+    if do_ast:
+        tprint(ast_node)
+        tpr_ast1 = time.perf_counter()
     print(rf'Tokens            done in {(tn1 - tn0) * 1000:.2f}ms')
     print(rf'Tokens_print      done in {(tpr_tk1 - tpr_tk0) * 1000:.2f}ms')
     print(rf'CST               done in {(tp1 - tp0) * 1000:.2f}ms')
     print(rf'CST_print         done in {(tpr_cst1 - tpr_cst0) * 1000:.2f}ms')
+    if do_ast:
+        print(rf'AST               done in {(ta1 - ta0) * 1000:.2f}ms')
+        print(rf'AST_print         done in {(tpr_ast1 - tpr_ast0) * 1000:.2f}ms')
 
 
 def main():
-    run(readfile('main_example_0.st'), 0)
+    run(readfile('main_example_0.st'), 0, do_ast=False)
     run(readfile('main_example_1.st'), 1)
 
 

main.py

@@ -0,0 +1,181 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from enum import Enum
+
+from ..common import HasRegion, StrRegion
+
+__all__ = [
+    "AstNode", "AstProgramNode", "VarDeclType", "AstDeclNode", "AstRepeat",
+    "AstIf", "AstWhile", "AstAssign", "AstAugAssign", "AstDefine", "AstNumber",
+    "AstString", "AstAnyName", "AstIdent", "AstAttrName", "AstAttribute",
+    "AstItem", "AstCall", "AstOp", "AstBinOp", "AstUnaryOp",
+]
+
+
+@dataclass
+class AstNode(HasRegion):
+    region: StrRegion
+    name = None  # type: str
+    del name  # So we get better error msg if we forget to add it to a class
+
+
+@dataclass
+class AstProgramNode(AstNode):
+    name = 'program'
+    statements: list[AstNode]
+
+
+# region ---- <Statements> ----
+class VarDeclType(Enum):
+    LET = 'let'
+    GLOBAL = 'global'
+
+
+@dataclass
+class AstDeclNode(AstNode):
+    name = 'var_decl'
+    type: VarDeclType
+    decls: list[tuple[AstIdent, AstNode | None]]
+
+
+@dataclass
+class AstRepeat(AstNode):
+    name = 'repeat'
+    count: AstNode
+    body: list[AstNode]
+
+
+@dataclass
+class AstIf(AstNode):
+    name = 'if'
+    cond: AstNode
+    if_body: list[AstNode]
+    # elseif = else{if
+    else_body: list[AstNode] | None = None
+    # ^ Separate cases for no block and empty block (can be else {} to easily
+    # add extra blocks in scratch interface)
+
+
+@dataclass
+class AstWhile(AstNode):
+    name = 'while'
+    cond: AstNode
+    body: list[AstNode]
+
+
+@dataclass
+class AstAssign(AstNode):
+    name = '='
+    target: AstNode
+    source: AstNode
+
+
+@dataclass
+class AstAugAssign(AstNode):
+    op: str  # maybe attach a StrRegion to the location of the op??
+    target: AstNode
+    source: AstNode
+
+    @property
+    def name(self):
+        return self.op
+
+
+@dataclass
+class AstDefine(AstNode):
+    name = 'def'
+
+    ident: AstIdent
+    params: list[tuple[AstIdent, AstIdent]]  # type, ident
+    body: list[AstNode]
+# endregion ---- </Statements> ----
+
+
+# region ---- <Expressions> ----
+@dataclass
+class AstNumber(AstNode):
+    # No real point in storing the string representation (could always StrRegion.resolve())
+    value: float | int
+
+
+@dataclass
+class AstString(AstNode):
+    value: str  # Values with escapes, etc. resolved
+
+
+@dataclass
+class AstAnyName(AstNode):
+    id: str
+
+    def __post_init__(self):
+        if type(self) == AstAnyName:
+            raise TypeError("AstAnyName must not be instantiated directly.")
+
+
+@dataclass
+class AstIdent(AstAnyName):
+    name = 'ident'
+
+
+@dataclass
+class AstAttrName(AstAnyName):
+    name = 'attr'
+
+
+@dataclass
+class AstAttribute(AstNode):
+    name = '.'
+    obj: AstNode
+    attr: AstAttrName
+
+
+@dataclass
+class AstItem(AstNode):
+    name = 'item'
+    obj: AstNode
+    index: AstNode
+
+
+@dataclass
+class AstCall(AstNode):
+    name = 'call'
+    obj: AstNode
+    args: list[AstNode]
+
+
+@dataclass
+class AstOp(AstNode):
+    op: str
+
+
+@dataclass
+class AstBinOp(AstOp):
+    left: AstNode
+    right: AstNode
+
+    valid_ops = [*'+-*/%', '**', '..', '||', '&&',  # ops
+                 '==', '!=', '<', '>', '<=', '>='  # comparisons
+                 ]  # type: list[str]
+
+    def __post_init__(self):
+        assert self.op in self.valid_ops
+
+    @property
+    def name(self):
+        return self.op
+
+
+@dataclass
+class AstUnaryOp(AstOp):
+    operand: AstNode
+
+    valid_ops = ('+', '-', '!')
+
+    def __post_init__(self):
+        assert self.op in self.valid_ops
+
+    @property
+    def name(self):
+        return self.op
+# endregion ---- </Expressions> ----