Merge branch 'parser-module'

Author: Maxcode123

Makefile

@@ -1,5 +1,5 @@
 clean:
-	rm -rf src/syntactes/__pycache__ src/syntactes/tests/__pycache__
+	rm -rf src/syntactes/__pycache__ src/syntactes/tests/__pycache__ src/syntactes/parser/__pycache__
 	rm -rf dist src/syntactes.egg-info
 
 test:

README.md

@@ -8,6 +8,8 @@
 Python parser generator
 
 ## Quick start
+
+### Creating a parsing table
 ```py
 from syntactes import Grammar, Rule, SLRGenerator, Token
 
@@ -54,16 +56,86 @@ SLR PARSING TABLE
 -------------------------------------------------
 |     |  $   |  +   |  E   |  S   |  T   |  x  |
 -------------------------------------------------
-|  1  |  --  |  --  |  s3  |  --  |  s4  |  s2 |
+|  1  |  --  |  --  |  s4  |  --  |  s2  |  s3 |
 -------------------------------------------------
-|  2  |  r4  |  r4  |  --  |  --  |  --  |  -- |
+|  2  |  r2  |  s5  |  --  |  --  |  --  |  -- |
 -------------------------------------------------
-|  3  |  a  |  --  |  --  |  --  |  --  |  -- |
-------------------------------------------------
-|  4  |  r2  |  s5  |  --  |  --  |  --  |  -- |
+|  3  |  r4  |  r4  |  --  |  --  |  --  |  -- |
 -------------------------------------------------
-|  5  |  --  |  --  |  s6  |  --  |  s4  |  s2 |
+|  4  |  a  |  --  |  --  |  --  |  --  |  -- |
+------------------------------------------------
+|  5  |  --  |  --  |  s6  |  --  |  s2  |  s3 |
 -------------------------------------------------
 |  6  |  r1  |  --  |  --  |  --  |  --  |  -- |
 -------------------------------------------------
 ```
+
+### Parsing
+
+```py
+from syntactes import Grammar, Rule, Token
+from syntactes.parser import ParserError, SLRParser, execute_on
+
+EOF = Token.eof()
+S = Token("S", is_terminal=False)
+E = Token("E", False)
+T = Token("T", False)
+x = Token("x", True, 1)  # value of token is 1
+PLUS = Token("+", True)
+
+tokens = {EOF, S, E, T, x, PLUS}
+
+# 0. S -> E $
+# 1. E -> T + E
+# 2. E -> T
+# 3. T -> x
+rule_1 = Rule(0, S, E, EOF)
+rule_2 = Rule(1, E, T, PLUS, E)
+rule_3 = Rule(2, E, T)
+rule_4 = Rule(4, T, x)
+
+rules = (rule_1, rule_2, rule_3, rule_4)
+
+grammar = Grammar(rule_1, rules, tokens)
+
+parser = SLRParser.from_grammar(grammar)
+
+
+@execute_on(rule_4)
+def push_value(x_token):
+    # Add and argument for every token on the right-hand side of the rule.
+    print(
+        f"received token {x_token} with value: {x_token.value}, reducing by rule: {rule_4}"
+    )
+
+
+@execute_on(rule_2)
+def add(left, plus, right):
+    print(f"received tokens {left}, {plus}, {right}, reducing by rule: {rule_2}")
+
+
+print("Parsing stream: x + x + x $\n")
+parser.parse([x, PLUS, x, PLUS, x, EOF])
+
+print("\nParsing stream: x + $\n")
+try:
+    parser.parse([x, PLUS, EOF])
+except ParserError as e:
+    print("ParserError:", e)
+```
+
+Running the above example produces this output:
+```
+Parsing stream: x + x + x $
+
+received token x with value: 1, reducing by rule: T -> x
+received token x with value: 1, reducing by rule: T -> x
+received token x with value: 1, reducing by rule: T -> x
+received tokens E, +, T, reducing by rule: E -> T + E
+received tokens E, +, T, reducing by rule: E -> T + E
+
+Parsing stream: x + $
+
+received token x with value: 1, reducing by rule: T -> x
+ParserError: Received token: $; expected one of: ['x', 'T', 'E']
+```

examples/parser.py

@@ -0,0 +1,49 @@
+from syntactes import Grammar, Rule, Token
+from syntactes.parser import ParserError, SLRParser, execute_on
+
+EOF = Token.eof()
+S = Token("S", is_terminal=False)
+E = Token("E", False)
+T = Token("T", False)
+x = Token("x", True, 1)  # value of token is 1
+PLUS = Token("+", True)
+
+tokens = {EOF, S, E, T, x, PLUS}
+
+# 0. S -> E $
+# 1. E -> T + E
+# 2. E -> T
+# 3. T -> x
+rule_1 = Rule(0, S, E, EOF)
+rule_2 = Rule(1, E, T, PLUS, E)
+rule_3 = Rule(2, E, T)
+rule_4 = Rule(4, T, x)
+
+rules = (rule_1, rule_2, rule_3, rule_4)
+
+grammar = Grammar(rule_1, rules, tokens)
+
+parser = SLRParser.from_grammar(grammar)
+
+
+@execute_on(rule_4)
+def push_value(x_token):
+    # Add and argument for every token on the right-hand side of the rule.
+    print(
+        f"received token {x_token} with value: {x_token.value}, reducing by rule: {rule_4}"
+    )
+
+
+@execute_on(rule_2)
+def add(left, plus, right):
+    print(f"received tokens {left}, {plus}, {right}, reducing by rule: {rule_2}")
+
+
+print("Parsing stream: x + x + x $\n")
+parser.parse([x, PLUS, x, PLUS, x, EOF])
+
+print("\nParsing stream: x + $\n")
+try:
+    parser.parse([x, PLUS, EOF])
+except ParserError as e:
+    print("ParserError:", e)

example.py examples/parsing_table.pyexample.py renamed to examples/parsing_table.py

@@ -1,4 +1,5 @@
-from .generator import LR0Generator, SLRGenerator
-from .grammar import Grammar
-from .rule import Rule
 from .token import Token
+from .rule import Rule
+from .grammar import Grammar
+from .generator import LR0Generator, SLRGenerator
+from .table import LR0ParsingTable, SLRParsingTable

src/syntactes/__init__.py

@@ -43,6 +43,27 @@ def __init__(self, actionable: Actionable, action_type: ActionType) -> None:
         self.actionable = actionable
         self.action_type = action_type
 
+    @staticmethod
+    def shift(state: Actionable) -> "Action":
+        """
+        Create a shift action to the given state.
+        """
+        return Action(state, ActionType.SHIFT)
+
+    @staticmethod
+    def reduce(rule: Actionable) -> "Action":
+        """
+        Create a reduce action of the given rule.
+        """
+        return Action(rule, ActionType.REDUCE)
+
+    @staticmethod
+    def accept() -> "Action":
+        """
+        Create an accept action.
+        """
+        return Action(None, ActionType.ACCEPT)
+
     def __repr__(self) -> str:
         return f"<Action: {self}>"
 

src/syntactes/_action.py

@@ -1,21 +1,8 @@
-from typing import Iterable, Protocol
+from typing import Iterable
 
 from syntactes._item import LR0Item
 
 
-class State(Protocol):
-    """
-    State of parser automaton. A state is a set of items.
-    """
-
-    items: set
-
-    def __repr__(self) -> str: ...
-    def __str__(self) -> str: ...
-    def __hash__(self) -> int: ...
-    def __eq__(self) -> bool: ...
-
-
 class LR0State:
     """
     State of LR0 parser. A LR0 state is a set of LR0 items.
@@ -24,6 +11,7 @@ class LR0State:
     def __init__(self) -> None:
         self.number = None
         self.items = set()
+        self.is_final = False
 
     @staticmethod
     def from_items(items: Iterable[LR0Item]) -> "LR0State":
@@ -44,6 +32,9 @@ def add_item(self, item: LR0Item) -> None:
     def set_number(self, number: int) -> None:
         self.number = number
 
+    def set_final(self) -> None:
+        self.is_final = True
+
     def __repr__(self) -> str:
         return f"<LR0State: {self.number}>"
 

src/syntactes/_state.py

@@ -1,9 +1,8 @@
+from syntactes import Grammar, Token
 from syntactes._action import Action, ActionType
 from syntactes._item import LR0Item
 from syntactes._state import LR0State
-from syntactes.grammar import Grammar
 from syntactes.table import Entry, LR0ParsingTable, SLRParsingTable
-from syntactes.token import Token
 
 
 class LR0Generator:
@@ -191,7 +190,11 @@ def _extend_states_and_shift_entries(
         EOF = Token.eof()
         for state in states:
             for item in state.items:
-                if item.dot_is_last() or item.after_dot == EOF:
+                if item.dot_is_last():
+                    continue
+
+                if item.after_dot == EOF:
+                    state.set_final()
                     continue
 
                 new_items = self.goto(state.items, item.after_dot)
@@ -204,8 +207,7 @@ def _extend_states_and_shift_entries(
                 number = _states.setdefault(new, len(_states) + 1)
                 new.set_number(number)
 
-                action = Action(new, ActionType.SHIFT)
-                _entries.add(Entry(state, item.after_dot, action))
+                _entries.add(Entry(state, item.after_dot, Action.shift(new)))
 
         return _states, _entries
 
@@ -218,16 +220,14 @@ def _create_reduce_entries(self, states: set[LR0State]) -> set[Entry]:
         for state in states:
             for item in state.items:
                 if item.after_dot == Token.eof():
-                    action = Action(item.rule, ActionType.ACCEPT)
-                    entries.add(Entry(state, Token.eof(), action))
+                    entries.add(Entry(state, Token.eof(), Action.accept()))
 
                 if not item.dot_is_last():
                     continue
 
-                action = Action(item.rule, ActionType.REDUCE)
                 for token in self.grammar.tokens:
                     if token.is_terminal:
-                        entries.add(Entry(state, token, action))
+                        entries.add(Entry(state, token, Action.reduce(item.rule)))
 
         return entries
 
@@ -255,14 +255,12 @@ def _create_reduce_entries(self, states: set[LR0State]) -> set[Entry]:
         for state in states:
             for item in state.items:
                 if item.after_dot == Token.eof():
-                    action = Action(item.rule, ActionType.ACCEPT)
-                    entries.add(Entry(state, Token.eof(), action))
+                    entries.add(Entry(state, Token.eof(), Action.accept()))
 
                 if not item.dot_is_last():
                     continue
 
-                action = Action(item.rule, ActionType.REDUCE)
                 for token in self._follow(item.rule.lhs):
-                    entries.add(Entry(state, token, action))
+                    entries.add(Entry(state, token, Action.reduce(item.rule)))
 
         return entries

src/syntactes/generator.py

@@ -1,7 +1,6 @@
 from typing import Iterable
 
-from syntactes.rule import Rule
-from syntactes.token import Token
+from syntactes import Rule, Token
 
 
 class Grammar:

src/syntactes/grammar.py

@@ -0,0 +1,3 @@
+from .exception import NotAcceptedError, ParserError, UnexpectedTokenError
+from .execute import ExecutablesRegistry, execute_on
+from .parser import LR0Parser, SLRParser