|
1 | 1 | from __future__ import annotations |
2 | 2 |
|
3 | | -import io |
4 | 3 | from collections import deque |
5 | | -from typing import Callable, Deque, Generator, Generic, Iterable, TypeVar, Union |
| 4 | +from typing import Callable, Deque, Generator, Generic, Iterable, NamedTuple, TypeVar |
| 5 | + |
| 6 | +from ._time import get_time |
6 | 7 |
|
7 | 8 |
|
8 | 9 | class ParseError(Exception): |
9 | | - pass |
| 10 | + """Base class for parse related errors.""" |
10 | 11 |
|
11 | 12 |
|
12 | 13 | class ParseEOF(ParseError): |
13 | 14 | """End of Stream.""" |
14 | 15 |
|
15 | 16 |
|
16 | | -class Awaitable: |
17 | | - __slots__: list[str] = [] |
18 | | - |
19 | | - |
20 | | -class _Read(Awaitable): |
21 | | - __slots__ = ["remaining"] |
22 | | - |
23 | | - def __init__(self, count: int) -> None: |
24 | | - self.remaining = count |
| 17 | +class ParseTimeout(ParseError): |
| 18 | + """Read has timed out.""" |
25 | 19 |
|
26 | | - def __repr__(self) -> str: |
27 | | - return f"_ReadBytes({self.remaining})" |
28 | 20 |
|
| 21 | +class Read1(NamedTuple): |
| 22 | + """Reads a single character.""" |
29 | 23 |
|
30 | | -class _Read1(Awaitable): |
31 | | - __slots__: list[str] = [] |
| 24 | + timeout: float | None = None |
| 25 | + """Optional timeout in seconds.""" |
32 | 26 |
|
33 | 27 |
|
34 | | -class _ReadUntil(Awaitable): |
35 | | - __slots__ = ["sep", "max_bytes"] |
| 28 | +class Peek1(NamedTuple): |
| 29 | + """Reads a single character, but does not advance the parser position.""" |
36 | 30 |
|
37 | | - def __init__(self, sep: str, max_bytes: int | None = None) -> None: |
38 | | - self.sep = sep |
39 | | - self.max_bytes = max_bytes |
40 | | - |
41 | | - |
42 | | -class _PeekBuffer(Awaitable): |
43 | | - __slots__: list[str] = [] |
| 31 | + timeout: float | None = None |
| 32 | + """Optional timeout in seconds.""" |
44 | 33 |
|
45 | 34 |
|
46 | 35 | T = TypeVar("T") |
47 | | - |
48 | | - |
49 | 36 | TokenCallback = Callable[[T], None] |
50 | 37 |
|
51 | 38 |
|
52 | 39 | class Parser(Generic[T]): |
53 | | - read = _Read |
54 | | - read1 = _Read1 |
55 | | - read_until = _ReadUntil |
56 | | - peek_buffer = _PeekBuffer |
| 40 | + """Base class for a simple parser.""" |
| 41 | + |
| 42 | + read1 = Read1 |
| 43 | + peek1 = Peek1 |
57 | 44 |
|
58 | 45 | def __init__(self) -> None: |
59 | | - self._buffer = io.StringIO() |
60 | 46 | self._eof = False |
61 | 47 | self._tokens: Deque[T] = deque() |
62 | 48 | self._gen = self.parse(self._tokens.append) |
63 | | - self._awaiting: Union[Awaitable, T] = next(self._gen) |
| 49 | + self._awaiting: Read1 | Peek1 = next(self._gen) |
| 50 | + self._timeout_time: float | None = None |
64 | 51 |
|
65 | 52 | @property |
66 | 53 | def is_eof(self) -> bool: |
| 54 | + """Is the parser at the end of the file (i.e. exhausted)?""" |
67 | 55 | return self._eof |
68 | 56 |
|
69 | | - def reset(self) -> None: |
70 | | - self._gen = self.parse(self._tokens.append) |
71 | | - self._awaiting = next(self._gen) |
| 57 | + def tick(self) -> Iterable[T]: |
| 58 | + """Call at regular intervals to check for timeouts.""" |
| 59 | + if self._timeout_time is not None and get_time() >= self._timeout_time: |
| 60 | + self._timeout_time = None |
| 61 | + self._awaiting = self._gen.throw(ParseTimeout()) |
| 62 | + while self._tokens: |
| 63 | + yield self._tokens.popleft() |
72 | 64 |
|
73 | 65 | def feed(self, data: str) -> Iterable[T]: |
| 66 | + """Feed data to be parsed. |
| 67 | +
|
| 68 | + Args: |
| 69 | + data: Data to parser. |
| 70 | +
|
| 71 | + Raises: |
| 72 | + ParseError: If the data could not be parsed. |
| 73 | +
|
| 74 | + Yields: |
| 75 | + T: A generic data type. |
| 76 | + """ |
74 | 77 | if self._eof: |
75 | 78 | raise ParseError("end of file reached") from None |
| 79 | + |
| 80 | + tokens = self._tokens |
| 81 | + popleft = tokens.popleft |
| 82 | + |
76 | 83 | if not data: |
77 | 84 | self._eof = True |
78 | 85 | try: |
79 | | - self._gen.send(self._buffer.getvalue()) |
| 86 | + self._gen.throw(EOFError()) |
80 | 87 | except StopIteration: |
81 | | - raise ParseError("end of file reached") from None |
82 | | - while self._tokens: |
83 | | - yield self._tokens.popleft() |
84 | | - |
85 | | - self._buffer.truncate(0) |
| 88 | + pass |
| 89 | + while tokens: |
| 90 | + yield popleft() |
86 | 91 | return |
87 | 92 |
|
88 | | - _buffer = self._buffer |
89 | 93 | pos = 0 |
90 | | - tokens = self._tokens |
91 | | - popleft = tokens.popleft |
92 | 94 | data_size = len(data) |
93 | 95 |
|
94 | 96 | while tokens: |
95 | 97 | yield popleft() |
96 | 98 |
|
97 | | - while pos < data_size or isinstance(self._awaiting, _PeekBuffer): |
| 99 | + while pos < data_size: |
98 | 100 | _awaiting = self._awaiting |
99 | | - if isinstance(_awaiting, _Read1): |
100 | | - self._awaiting = self._gen.send(data[pos : pos + 1]) |
| 101 | + if isinstance(_awaiting, Read1): |
| 102 | + self._timeout_time = None |
| 103 | + self._awaiting = self._gen.send(data[pos]) |
101 | 104 | pos += 1 |
| 105 | + elif isinstance(_awaiting, Peek1): |
| 106 | + self._timeout_time = None |
| 107 | + self._awaiting = self._gen.send(data[pos]) |
102 | 108 |
|
103 | | - elif isinstance(_awaiting, _PeekBuffer): |
104 | | - self._awaiting = self._gen.send(data[pos:]) |
105 | | - |
106 | | - elif isinstance(_awaiting, _Read): |
107 | | - remaining = _awaiting.remaining |
108 | | - chunk = data[pos : pos + remaining] |
109 | | - chunk_size = len(chunk) |
110 | | - pos += chunk_size |
111 | | - _buffer.write(chunk) |
112 | | - remaining -= chunk_size |
113 | | - if remaining: |
114 | | - _awaiting.remaining = remaining |
115 | | - else: |
116 | | - _awaiting = self._gen.send(_buffer.getvalue()) |
117 | | - _buffer.seek(0) |
118 | | - _buffer.truncate() |
119 | | - |
120 | | - elif isinstance(_awaiting, _ReadUntil): |
121 | | - chunk = data[pos:] |
122 | | - _buffer.write(chunk) |
123 | | - sep = _awaiting.sep |
124 | | - sep_index = _buffer.getvalue().find(sep) |
125 | | - |
126 | | - if sep_index == -1: |
127 | | - pos += len(chunk) |
128 | | - if ( |
129 | | - _awaiting.max_bytes is not None |
130 | | - and _buffer.tell() > _awaiting.max_bytes |
131 | | - ): |
132 | | - self._gen.throw(ParseError(f"expected {sep}")) |
133 | | - else: |
134 | | - sep_index += len(sep) |
135 | | - if ( |
136 | | - _awaiting.max_bytes is not None |
137 | | - and sep_index > _awaiting.max_bytes |
138 | | - ): |
139 | | - self._gen.throw(ParseError(f"expected {sep}")) |
140 | | - data = _buffer.getvalue()[sep_index:] |
141 | | - pos = 0 |
142 | | - self._awaiting = self._gen.send(_buffer.getvalue()[:sep_index]) |
143 | | - _buffer.seek(0) |
144 | | - _buffer.truncate() |
| 109 | + if self._awaiting.timeout is not None: |
| 110 | + self._timeout_time = get_time() + self._awaiting.timeout |
145 | 111 |
|
146 | 112 | while tokens: |
147 | 113 | yield popleft() |
148 | 114 |
|
149 | | - def parse(self, on_token: Callable[[T], None]) -> Generator[Awaitable, str, None]: |
150 | | - yield from () |
151 | | - |
| 115 | + def parse( |
| 116 | + self, token_callback: TokenCallback |
| 117 | + ) -> Generator[Read1 | Peek1, str, None]: |
| 118 | + """Implement to parse a stream of text. |
152 | 119 |
|
153 | | -if __name__ == "__main__": |
154 | | - data = "Where there is a Will there is a way!" |
| 120 | + Args: |
| 121 | + token_callback: Callable to report a successful parsed data type. |
155 | 122 |
|
156 | | - class TestParser(Parser[str]): |
157 | | - def parse( |
158 | | - self, on_token: Callable[[str], None] |
159 | | - ) -> Generator[Awaitable, str, None]: |
160 | | - while True: |
161 | | - data = yield self.read1() |
162 | | - if not data: |
163 | | - break |
164 | | - on_token(data) |
165 | | - |
166 | | - test_parser = TestParser() |
167 | | - |
168 | | - for n in range(0, len(data), 5): |
169 | | - for token in test_parser.feed(data[n : n + 5]): |
170 | | - print(token) |
171 | | - for token in test_parser.feed(""): |
172 | | - print(token) |
| 123 | + Yields: |
| 124 | + ParseAwaitable: One of `self.read1` or `self.peek1` |
| 125 | + """ |
| 126 | + yield from () |
0 commit comments