Merge pull request #38 from MarcellPerger1/start-ast

Refactor tokenizer

Author: MarcellPerger1

parser/common/__init__.py

@@ -1,4 +1,7 @@
 from .common import *
 from .error import *  # <^ might add some stuff so `import *`
 from .str_region import StrRegion  # <-- won't add stuff to str_region so not `import *`
-from .tree_print import *
+# IMPORTant: don't include tree_print here as that causes circular import issue:
+#  - lexer.tokens imports ..common (for StrRegion)
+#  - tree_print also loaded from common/__init__.py
+#  - tree_print needs `Node`... and `Node` needs lexer.tokens

parser/cst/base_node.py

@@ -1,9 +1,12 @@
 from __future__ import annotations
 
 from dataclasses import dataclass, field
+from typing import TYPE_CHECKING
 
 from ..common import StrRegion, HasRegion
-from ..tokens import Token
+
+if TYPE_CHECKING:
+    from ..tokens import Token
 
 
 @dataclass

parser/lexer/__init__.py

@@ -1,5 +1,5 @@
-from .tokenizer import (
-    Tokenizer,
-    TokenizerError, LocatedTokenizerError,
-    LocatedMalformedNumberError, MalformedNumberError,
-    print_tokens, format_tokens)
+from .tokenizer import Tokenizer
+from .token_print import print_tokens, format_tokens
+from .errors import (
+    TokenizerError, LocatedTokenizerError, MalformedNumberError,
+    LocatedMalformedNumberError)

parser/lexer/errors.py

@@ -0,0 +1,19 @@
+from __future__ import annotations
+
+from parser.common import BaseParseError, BaseLocatedError
+
+
+class TokenizerError(BaseParseError):
+    ...
+
+
+class LocatedTokenizerError(BaseLocatedError, TokenizerError):
+    ...
+
+
+class MalformedNumberError(TokenizerError):
+    ...
+
+
+class LocatedMalformedNumberError(LocatedTokenizerError, MalformedNumberError):
+    ...

parser/lexer/number_parser.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from string import digits
+
+from .errors import LocatedMalformedNumberError
+from .src_handler import UsesSrc
+from .tokens import Token, NumberToken
+from ..common import StrRegion
+
+
+class NumberParser(UsesSrc):
+    default_err_type = LocatedMalformedNumberError
+
+    # todo 0x, 0b (I refuse to add octal literals) - also hex floats???
+    def _parse_digit_seq(self, start: int) -> int | None:
+        # (Returns None if no digits)
+        idx = start
+        if self.get(idx) == '_':
+            raise self.err("Can't have '_' at the start of a number", idx)
+        if self.get(idx) not in digits:
+            return None
+        idx += 1
+        while True:
+            if self.get(idx) == '_':
+                if self.get(idx + 1) in digits:
+                    idx += 2  # '_' and digit
+                elif self.get(idx + 1) == '_':
+                    raise self.err(
+                        "Can only have one consecutive '_' in a number", idx + 1)
+                else:
+                    raise self.err(
+                        "Can't have '_' at the end of a number", idx)
+            elif self.get(idx) in digits:
+                idx += 1
+            else:
+                return idx  # end of digits/'_'
+
+    def _parse_num_no_exp(self, idx: int) -> int:
+        new_idx = self._parse_digit_seq(idx)
+        if new_idx is None:
+            if self.get(idx) != '.':
+                raise self.err("Number must start with digit or '.' ", idx)
+            has_pre_dot = False
+        else:
+            has_pre_dot = True
+            idx = new_idx
+        if self.get(idx) != '.':
+            # eg: 1234, 567e-5, 8 +9-10
+            return idx
+        idx += 1
+        new_idx = self._parse_digit_seq(idx)
+        if new_idx is None:
+            has_post_dot = False
+        else:
+            has_post_dot = True
+            idx = new_idx
+        if has_pre_dot or has_post_dot:
+            return idx
+        raise self.err("Number cannot be a single '.' "
+                       "(expected digits before or after)", idx)
+
+    def _parse_number(self, idx: int) -> int:
+        idx = self._parse_num_no_exp(idx)
+        if self.get(idx).lower() != 'e':
+            return idx
+        idx += 1
+        # need to handle '-' here explicitly as it is part of the number
+        # so can't just be parsed as a separate operator
+        if self.get(idx) == '-':
+            idx += 1
+        new_idx = self._parse_digit_seq(idx)  # no dot after the 'e'
+        if new_idx is None:
+            # eg: 1.2eC, 8e-Q which is always an error
+            raise self.err("Expected integer after <number>e", idx)
+        idx = new_idx
+        return idx
+
+    def parse(self, start: int) -> Token:
+        return NumberToken(StrRegion(start, self._parse_number(start)))

parser/lexer/src_handler.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Sequence
+
+from .errors import LocatedTokenizerError
+from .tokens import Token
+from ..common import StrRegion, BaseLocatedError, region_union
+
+
+class UsesSrc:
+    def __init__(self, src: str):
+        self.src: str = src
+
+    def __getitem__(self, item: int | slice) -> str:
+        return self.src[item]
+
+    def eof(self, idx: int):
+        return idx >= len(self.src)
+
+    def get(self, idx: int, eof: str = '\0') -> str:
+        try:
+            return self.src[idx]
+        except IndexError:
+            return eof
+
+    default_err_type = LocatedTokenizerError
+
+    def err(self, msg: str,
+            loc: int | Token | StrRegion | Sequence[int | Token | StrRegion],
+            tp: type[BaseLocatedError] = None):
+        try:
+            seq: tuple[int | Token | StrRegion, ...] = tuple(loc)
+        except TypeError:
+            seq = (loc,)
+        region = region_union([
+            StrRegion(o, o + 1) if isinstance(o, int) else o
+            for o in seq])
+        tp = tp or self.default_err_type
+        return tp(msg, region, self.src)

parser/lexer/token_print.py

@@ -0,0 +1,29 @@
+from __future__ import annotations
+
+import sys
+from io import StringIO
+from typing import IO
+
+from .tokens import Token
+
+
+def print_tokens(src: str, tokens: list[Token], stream: IO[str] = None, do_ws=False):
+    if stream is None:
+        stream = sys.stdout
+    table = []
+    for tok in tokens:
+        if tok.is_whitespace:
+            if do_ws:
+                table.append(['(WS) ' + repr(tok.region.resolve(src)), tok.name])
+        else:
+            table.append([str(tok.region.resolve(src)), tok.name])
+    max0 = max(len(r[0]) for r in table)
+    max1 = max(len(r[1]) for r in table)
+    for s0, s1 in table:
+        print(f'{s0:>{max0}} | {s1:>{max1}}', file=stream)
+
+
+def format_tokens(src: str, tokens: list[Token], do_ws=False):
+    out = StringIO()
+    print_tokens(src, tokens, out, do_ws)
+    return out.getvalue()