refactor(tokenizer): Extract NumberParser into own file

Author: MarcellPerger1

parser/lexer/number_parser.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from string import digits
+
+from parser.common import StrRegion
+from parser.lexer import LocatedMalformedNumberError
+from parser.lexer.src_handler import UsesSrc
+from parser.lexer.tokens import Token, NumberToken
+
+
+class NumberParser(UsesSrc):
+    default_err_type = LocatedMalformedNumberError
+
+    # todo 0x, 0b (I refuse to add octal literals) - also hex floats???
+    def _parse_digit_seq(self, start: int) -> int | None:
+        # (Returns None if no digits)
+        idx = start
+        if self.get(idx) == '_':
+            raise self.err("Can't have '_' at the start of a number", idx)
+        if self.get(idx) not in digits:
+            return None
+        idx += 1
+        while True:
+            if self.get(idx) == '_':
+                if self.get(idx + 1) in digits:
+                    idx += 2  # '_' and digit
+                elif self.get(idx + 1) == '_':
+                    raise self.err(
+                        "Can only have one consecutive '_' in a number", idx + 1)
+                else:
+                    raise self.err(
+                        "Can't have '_' at the end of a number", idx)
+            elif self.get(idx) in digits:
+                idx += 1
+            else:
+                return idx  # end of digits/'_'
+
+    def _parse_num_no_exp(self, idx: int) -> int:
+        new_idx = self._parse_digit_seq(idx)
+        if new_idx is None:
+            if self.get(idx) != '.':
+                raise self.err("Number must start with digit or '.' ", idx)
+            has_pre_dot = False
+        else:
+            has_pre_dot = True
+            idx = new_idx
+        if self.get(idx) != '.':
+            # eg: 1234, 567e-5, 8 +9-10
+            return idx
+        idx += 1
+        new_idx = self._parse_digit_seq(idx)
+        if new_idx is None:
+            has_post_dot = False
+        else:
+            has_post_dot = True
+            idx = new_idx
+        if has_pre_dot or has_post_dot:
+            return idx
+        raise self.err("Number cannot be a single '.' "
+                       "(expected digits before or after)", idx)
+
+    def _parse_number(self, idx: int) -> int:
+        idx = self._parse_num_no_exp(idx)
+        if self.get(idx).lower() != 'e':
+            return idx
+        idx += 1
+        # need to handle '-' here explicitly as it is part of the number
+        # so can't just be parsed as a separate operator
+        if self.get(idx) == '-':
+            idx += 1
+        new_idx = self._parse_digit_seq(idx)  # no dot after the 'e'
+        if new_idx is None:
+            # eg: 1.2eC, 8e-Q which is always an error
+            raise self.err("Expected integer after <number>e", idx)
+        idx = new_idx
+        return idx
+
+    def parse(self, start: int) -> Token:
+        return NumberToken(StrRegion(start, self._parse_number(start)))

parser/lexer/tokenizer.py

@@ -5,7 +5,7 @@
 from string import ascii_letters, digits
 from typing import IO
 
-from .errors import LocatedMalformedNumberError
+from .number_parser import NumberParser
 from .src_handler import UsesSrc
 from .tokens import *
 from ..common import StrRegion
@@ -170,7 +170,7 @@ def _t_number(self, idx: int) -> int:
         # doesn't handle negative numbers,
         # those should be handled as a separate '-' operator
         assert self[idx] != '-', "_t_number doesn't handle negative numbers"
-        return self.add_token(_IncrementalNumberParser(self.src).parse(idx))
+        return self.add_token(NumberParser(self.src).parse(idx))
 
     def _t_dot(self, idx: int) -> int:
         assert self[idx] == '.', "_t_dot should only be called if char is '.'"
@@ -222,77 +222,6 @@ def _t_ident_name(self, start: int) -> int:
 )
 
 
-class _IncrementalNumberParser(UsesSrc):
-    default_err_type = LocatedMalformedNumberError
-
-    # todo 0x, 0b (I refuse to add octal literals) - also hex floats???
-    def _parse_digit_seq(self, start: int) -> int | None:
-        # (Returns None if no digits)
-        idx = start
-        if self.get(idx) == '_':
-            raise self.err("Can't have '_' at the start of a number", idx)
-        if self.get(idx) not in digits:
-            return None
-        idx += 1
-        while True:
-            if self.get(idx) == '_':
-                if self.get(idx + 1) in digits:
-                    idx += 2  # '_' and digit
-                elif self.get(idx + 1) == '_':
-                    raise self.err(
-                        "Can only have one consecutive '_' in a number", idx + 1)
-                else:
-                    raise self.err(
-                        "Can't have '_' at the end of a number", idx)
-            elif self.get(idx) in digits:
-                idx += 1
-            else:
-                return idx  # end of digits/'_'
-
-    def _parse_num_no_exp(self, idx: int) -> int:
-        new_idx = self._parse_digit_seq(idx)
-        if new_idx is None:
-            if self.get(idx) != '.':
-                raise self.err("Number must start with digit or '.' ", idx)
-            has_pre_dot = False
-        else:
-            has_pre_dot = True
-            idx = new_idx
-        if self.get(idx) != '.':
-            # eg: 1234, 567e-5, 8 +9-10
-            return idx
-        idx += 1
-        new_idx = self._parse_digit_seq(idx)
-        if new_idx is None:
-            has_post_dot = False
-        else:
-            has_post_dot = True
-            idx = new_idx
-        if has_pre_dot or has_post_dot:
-            return idx
-        raise self.err("Number cannot be a single '.' "
-                       "(expected digits before or after)", idx)
-
-    def _parse_number(self, idx: int) -> int:
-        idx = self._parse_num_no_exp(idx)
-        if self.get(idx).lower() != 'e':
-            return idx
-        idx += 1
-        # need to handle '-' here explicitly as it is part of the number
-        # so can't just be parsed as a separate operator
-        if self.get(idx) == '-':
-            idx += 1
-        new_idx = self._parse_digit_seq(idx)  # no dot after the 'e'
-        if new_idx is None:
-            # eg: 1.2eC, 8e-Q which is always an error
-            raise self.err("Expected integer after <number>e", idx)
-        idx = new_idx
-        return idx
-
-    def parse(self, start: int) -> Token:
-        return NumberToken(StrRegion(start, self._parse_number(start)))
-
-
 def print_tokens(src: str, tokens: list[Token], stream: IO[str] = None, do_ws=False):
     if stream is None:
         stream = sys.stdout