Defines parser module

Author: Maxcode123

src/syntactes/parser/parser.py

@@ -0,0 +1,103 @@
+from collections import deque
+from typing import Iterable
+
+from syntactes import Token
+from syntactes._action import Action, ActionType
+from syntactes._state import LR0State
+from syntactes.parser import (
+    ExecutablesRegistry,
+    NotAcceptedError,
+    ParserError,
+    UnexpectedTokenError,
+)
+from syntactes.table import LR0ParsingTable, SLRParsingTable
+
+
+class LR0Parser:
+    """
+    Parses streams of tokens based on the configured parsing table.
+    """
+
+    def __init__(self, table: LR0ParsingTable) -> None:
+        self._table = table
+        self._token_stack: deque[Token] = deque()
+        self._state_stack: deque[LR0State] = deque()
+        self._token_stream: deque[Token] = deque()
+
+    def parse(self, stream: Iterable[Token]) -> None:
+        """
+        Parses the given stream of tokens. Expects the EOF token as the last one.
+
+        Raises `syntactes.parser.UnexpectedTokenError` if an unexpected token is
+        received.
+
+        Raises `syntactes.parser.NotAcceptedError` if the stream of token has been
+        parsed and the parser did not receive an accept action.
+        """
+        self._set_state(self._table.initial_state)
+        self._token_stream.extend(stream)
+
+        while len(self._token_stream) > 0:
+            token = self._token_stream.popleft()
+            self._apply_action(token, self._get_action(token))
+
+        if token != Token.eof():
+            self._raise(NotAcceptedError("Expected EOF token. "))
+
+        if not self._get_state().is_final:
+            actions = self._table.get(self._get_state())
+            expected_tokens = [] if actions is None else list(actions.keys())
+            self._raise(UnexpectedTokenError(Token.eof(), expected_tokens))
+
+    def _apply_action(self, token: Token, action: Action) -> None:
+        if action.action_type == ActionType.SHIFT:
+            self._token_stack.append(token)
+            self._set_state(action.actionable)
+        elif action.action_type == ActionType.REDUCE:
+            rule = action.actionable
+            args = [self._token_stack.pop() for _ in reversed(rule.rhs)]
+            self._token_stack.append(rule.lhs)
+
+            {self._state_stack.pop() for _ in rule.rhs}
+
+            executable = ExecutablesRegistry.get(rule)
+            executable(*args)
+
+            self._token_stream.appendleft(token)  # reduce actions do not consume tokenA
+
+            shift = self._get_action(rule.lhs)
+            self._set_state(shift.actionable)
+
+    def _get_action(self, token: Token) -> Action:
+        actions = self._table.get_actions(self._get_state(), token)
+        if actions is None:
+            actions = self._table.get(self._get_state())
+            expected_tokens = [] if actions is None else list(actions.keys())
+            self._raise(UnexpectedTokenError(token, expected_tokens))
+
+        action = self._resolve_conflict(actions)
+        return action
+
+    def _resolve_conflict(self, actions: list[Action]) -> Action:
+        return actions[0]
+
+    def _set_state(self, state: LR0State) -> None:
+        self._state_stack.append(state)
+
+    def _get_state(self) -> LR0State:
+        return self._state_stack[-1]
+
+    def _cleanup(self) -> None:
+        self._token_stack.clear()
+        self._state_stack.clear()
+        self._token_stream.clear()
+
+    def _raise(self, error: ParserError) -> None:
+        self._cleanup()
+        raise error from None
+
+
+class SLRParser(LR0Parser):
+    """
+    Parses streams of tokens based on the configured parsing table.
+    """

src/syntactes/tests/test_parser.py

@@ -0,0 +1,146 @@
+from unittest_extensions import TestCase, args
+
+from syntactes import Token
+from syntactes.parser import (
+    ExecutablesRegistry,
+    LR0Parser,
+    ParserError,
+    SLRParser,
+    execute_on,
+)
+from syntactes.tests.data import (
+    EOF,
+    PLUS,
+    lr0_parsing_table,
+    rule_2,
+    rule_4,
+    slr_parsing_table,
+    x,
+)
+
+x1 = Token("x", True, 1)
+x2 = Token("x", True, 2)
+
+
+class TestLR0Parser(TestCase):
+    def parser(self):
+        return self._parser
+
+    def setUp(self):
+        self._parser = LR0Parser(lr0_parsing_table())
+
+    def assert_parser_error(self):
+        self.assertResultRaises(ParserError)
+
+
+class TestLR0ParserParse(TestLR0Parser):
+    def subject(self, *stream):
+        return self.parser().parse(stream)
+
+    @args(x, EOF)
+    def test_simple_x(self):
+        self.result()
+
+    @args(x, PLUS, x, EOF)
+    def test_x_plus_x(self):
+        self.result()
+
+    @args(x)
+    def test_no_eof_raises(self):
+        self.assert_parser_error()
+
+    @args(x, x)
+    def test_x_x_raises(self):
+        self.assert_parser_error()
+
+    @args(x, PLUS)
+    def test_x_plus_raises(self):
+        self.assert_parser_error()
+
+    @args(x, PLUS, EOF)
+    def test_x_plus_eof_raises(self):
+        self.assert_parser_error()
+
+    @args(EOF)
+    def test_eof_raises(self):
+        self.assert_parser_error()
+
+
+class TestLR0ParserParseExecutables(TestLR0Parser):
+    def subject(self, *stream):
+        self.parser().parse(stream)
+        return self.sum
+
+    def add(self, _right, _plus, _left):
+        self.sum += 1
+
+    def setUp(self):
+        self.sum = 0
+        self.add = execute_on(rule_2)(self.add)
+        super().setUp()
+
+    @args(x, PLUS, x, EOF)
+    def test_x_plus_x(self):
+        self.assertResult(1)
+
+    @args(x, PLUS, x, PLUS, x, EOF)
+    def test_x_plus_x_plus_x(self):
+        self.assertResult(2)
+
+
+class TestLR0ParserParseExecutablesTokenValues(TestLR0Parser):
+    def subject(self, *stream):
+        self.parser().parse(stream)
+        return self.sum
+
+    def push(self, x):
+        self.stack.append(x.value)
+
+    def add(self, x1, _plus, x2):
+        self.sum = self.stack.pop() + self.stack.pop()
+
+    def setUp(self):
+        self.sum = 0
+        self.stack = list()
+        execute_on(rule_4)(self.push)
+        execute_on(rule_2)(self.add)
+        super().setUp()
+
+    def tearDown(self):
+        ExecutablesRegistry.clear()
+
+    @args(x1, PLUS, x1, EOF)
+    def test_x1_plus_x1(self):
+        self.assertResult(2)
+
+    @args(x1, PLUS, x2, EOF)
+    def test_x1_plus_x2(self):
+        self.assertResult(3)
+
+    @args(x2, PLUS, x2, EOF)
+    def test_x2_plus_x2(self):
+        self.assertResult(4)
+
+
+class TestSLRParser(TestCase):
+    def parser(self):
+        return self._parser
+
+    def setUp(self):
+        self._parser = SLRParser(slr_parsing_table())
+
+    def assert_parser_error(self):
+        self.assertResultRaises(ParserError)
+
+
+class TestSLRParserParse(TestSLRParser):
+    def subject(self, *stream):
+        return self.parser().parse(stream)
+
+    @args(x, x, EOF)
+    def test_x_x_eof_raises(self):
+        self.assert_parser_error()
+
+    @args(x, PLUS, x, EOF)
+    def test_x_plus_x(self):
+        self.result()