Skip to content

Commit 12278b5

Browse files
pengipengi
committed
Update foramtting and test coverage
1 parent 3c34e46 commit 12278b5

18 files changed

+714
-354
lines changed

‎.github/workflows/python-package.yml‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -27,14 +27,11 @@ jobs:
2727
- name: Install dependencies
2828
run: |
2929
python -m pip install --upgrade pip
30-
python -m pip install flake8 pytest
30+
python -m pip install flake8 pytest pytest-cov
3131
if [ -f requirements.txt ]; then pip install -r requirements.txt; fi
3232
- name: Lint with flake8
3333
run: |
34-
# stop the build if there are Python syntax errors or undefined names
35-
flake8 . --count --select=E9,F63,F7,F82 --show-source --statistics
36-
# exit-zero treats all errors as warnings. The GitHub editor is 127 chars wide
37-
flake8 . --count --exit-zero --max-complexity=10 --max-line-length=127 --statistics
34+
flake8
3835
- name: Test with pytest
3936
run: |
4037
pytest

‎.gitignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,3 +7,4 @@ __pycache__
77

88
/dist
99
*.egg-info
10+
.coverage

‎example.py‎

Lines changed: 51 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -1,112 +1,113 @@
11
import parsion
22
import re
33

4+
45
class ExprLang(parsion.Parsion):
56
SELF_CHECK = True
6-
7+
78
LEXER_RULES = [
8-
(None, r'(\s+)', lambda x: None),
9+
(None, r'(\s+)', lambda x: None),
910

1011
('STR', r'("(?:[^"\\]|\\.)*")',
1112
lambda x: re.sub(r'\\(.)', r'\1', x[1:-1])),
12-
('FLOAT', r'([0-9]+\.[0-9]*)', lambda x: float(x)),
13-
('INT', r'([0-9]+|0x[0-9a-fA-F]+)', lambda x: int(x,base=0)),
14-
15-
('FOR', r'(for)(?:[^a-z0-9_]|$)', lambda x: None),
16-
('IN', r'(in)(?:[^a-z0-9_]|$)', lambda x: None),
17-
('IF', r'(if)(?:[^a-z0-9_]|$)', lambda x: None),
18-
('THEN', r'(then)(?:[^a-z0-9_]|$)', lambda x: None),
19-
('ELSE', r'(else)(?:[^a-z0-9_]|$)', lambda x: None),
20-
21-
('NAME', r'([a-zA-Z0-9_]+)', lambda x: x),
22-
23-
('==', r'(==)', lambda x: None),
24-
25-
('-', r'(-)', lambda x: None),
26-
('+', r'(\+)', lambda x: None),
27-
('*', r'(\*)', lambda x: None),
28-
('/', r'(\/)', lambda x: None),
29-
30-
(',', r'(,)', lambda x: None),
31-
('.', r'(\.)', lambda x: None),
32-
('=', r'(=)', lambda x: None),
33-
('(', r'([\(])', lambda x: None),
34-
(')', r'([\)])', lambda x: None),
35-
('[', r'(\[)', lambda x: None),
36-
(']', r'(\])', lambda x: None),
13+
('FLOAT', r'([0-9]+\.[0-9]*)', lambda x: float(x)),
14+
('INT', r'([0-9]+|0x[0-9a-fA-F]+)', lambda x: int(x, base=0)),
15+
16+
('FOR', r'(for)(?:[^a-z0-9_]|$)', lambda x: None),
17+
('IN', r'(in)(?:[^a-z0-9_]|$)', lambda x: None),
18+
('IF', r'(if)(?:[^a-z0-9_]|$)', lambda x: None),
19+
('THEN', r'(then)(?:[^a-z0-9_]|$)', lambda x: None),
20+
('ELSE', r'(else)(?:[^a-z0-9_]|$)', lambda x: None),
21+
22+
('NAME', r'([a-zA-Z0-9_]+)', lambda x: x),
23+
24+
('==', r'(==)', lambda x: None),
25+
26+
('-', r'(-)', lambda x: None),
27+
('+', r'(\+)', lambda x: None),
28+
('*', r'(\*)', lambda x: None),
29+
('/', r'(\/)', lambda x: None),
30+
31+
(',', r'(,)', lambda x: None),
32+
('.', r'(\.)', lambda x: None),
33+
('=', r'(=)', lambda x: None),
34+
('(', r'([\(])', lambda x: None),
35+
(')', r'([\)])', lambda x: None),
36+
('[', r'(\[)', lambda x: None),
37+
(']', r'(\])', lambda x: None),
3738

3839
# Intended to not fail during lexing phase, but push invalid inputs
3940
# to parsing phase, with more useful error messages
40-
('CHAR', r'(.)', lambda x: x)
41+
('CHAR', r'(.)', lambda x: x)
4142
]
42-
43+
4344
GRAMMAR_RULES = [
4445
('entry', 'entry', 'expr'),
4546

4647
('expr_eq', 'expr', 'expr1 _== expr1'),
4748

4849
(None, 'expr', 'expr1'),
49-
50+
5051
('expr_add', 'expr1', 'expr1 _+ expr2'),
5152
('expr_sub', 'expr1', 'expr1 _- expr2'),
52-
53+
5354
(None, 'expr1', 'expr2'),
54-
55+
5556
('expr_mult', 'expr2', 'expr2 _* expr3'),
5657
('expr_div', 'expr2', 'expr2 _/ expr3'),
57-
58+
5859
(None, 'expr2', 'expr3'),
59-
60+
6061
('expr_neg', 'expr3', '_- expr4'),
61-
62+
6263
(None, 'expr3', 'expr4'),
63-
64+
6465
('expr_int', 'expr4', 'INT'),
6566
('expr_float', 'expr4', 'FLOAT'),
6667
('expr_var', 'expr4', 'NAME'),
6768
('expr_str', 'expr4', 'STR'),
68-
69+
6970
(None, 'expr4', '_( expr _)'),
7071
]
71-
72+
7273
def expr_eq(self, lhs, rhs):
7374
return lhs == rhs
74-
75+
7576
def expr_add(self, lhs, rhs):
7677
return lhs + rhs
77-
78+
7879
def expr_sub(self, lhs, rhs):
7980
return lhs - rhs
80-
81+
8182
def expr_mult(self, lhs, rhs):
8283
return lhs * rhs
83-
84+
8485
def expr_div(self, lhs, rhs):
8586
return lhs // rhs
86-
87+
8788
def expr_neg(self, v):
8889
return -v
89-
90+
9091
def expr_int(self, v):
9192
return v
92-
93+
9394
def expr_float(self, v):
9495
return v
95-
96+
9697
def expr_var(self, v):
9798
return v
98-
99+
99100
def expr_str(self, v):
100101
return v
101102

102103

103104
if __name__ == '__main__':
104105
# Generate the parser
105106
parser = ExprLang()
106-
107+
107108
# Print the parsing tables
108109
# This line uses "tabulate" library
109-
#parser.print()
110-
110+
# parser.print()
111+
111112
# Parse a string
112113
print(parser.parse('(12 + 32 * 4) / 7 + 13'))

‎parsion/__init__.py‎

Lines changed: 13 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,15 @@
11
from .core import Parsion, ParsionParseError
22
from .lex import ParsionLexerError, ParsionToken
3-
from .exceptions import *
3+
from .exceptions import ParsionException, ParsionGeneratorError, \
4+
ParsionInternalError, ParsionSelfCheckError
5+
6+
__all__ = [
7+
'Parsion',
8+
'ParsionParseError',
9+
'ParsionLexerError',
10+
'ParsionToken',
11+
'ParsionException',
12+
'ParsionGeneratorError',
13+
'ParsionInternalError',
14+
'ParsionSelfCheckError'
15+
]

‎parsion/core.py‎

Lines changed: 59 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -2,17 +2,34 @@
22
from parsion.parser import ParsionFSM
33
from .exceptions import ParsionInternalError
44

5+
56
class ParsionParseError(Exception):
67
pass
78

9+
810
class Parsion:
9-
LEXER_RULES=[]
10-
GRAMMAR_RULES=[]
11-
SELF_CHECK=True
12-
11+
LEXER_RULES = []
12+
GRAMMAR_RULES = []
13+
SELF_CHECK = True
14+
15+
STATIC_GRAMMAR = None
16+
STATIC_TABLE = None
17+
STATIC_ERROR_HANDLERS = None
18+
1319
def __init__(self, lex_rules=None, grammar_rules=None, self_check=True):
1420
self.lex = ParsionLexer(self.LEXER_RULES)
15-
self.parse_grammar, self.parse_table, self.error_handlers = ParsionFSM(self.GRAMMAR_RULES).export()
21+
22+
if self.STATIC_GRAMMAR is not None:
23+
self.parse_grammar = self.STATIC_GRAMMAR
24+
self.parse_table = self.STATIC_TABLE
25+
self.error_handlers = self.STATIC_ERROR_HANDLERS
26+
else:
27+
(
28+
self.parse_grammar,
29+
self.parse_table,
30+
self.error_handlers
31+
) = ParsionFSM(self.GRAMMAR_RULES).export()
32+
1633
if self.SELF_CHECK:
1734
self._self_check()
1835

@@ -28,50 +45,45 @@ def _call_reduce(self, goal, accepts, parts):
2845
return args[0]
2946
else:
3047
return getattr(self, goal)(*args)
31-
48+
3249
def _call_error_handler(self, handler, error_stack, error_tokens):
3350
return getattr(self, handler)(error_stack, error_tokens)
34-
51+
3552
def parse(self, input):
3653
tokens = [(tok.name, tok.value) for tok in self.lex.tokenize(input)]
3754
stack = [('START', 0)]
38-
55+
3956
while len(tokens) > 0:
4057
tok_name, tok_value = tokens[0]
4158
cur_state = stack[-1][1]
4259
if tok_name not in self.parse_table[cur_state]:
4360
# Unexpected token, do error recovery
44-
45-
# First, pop stack until error handler
46-
error_stack = []
47-
while len(stack) > 0 and stack[-1][1] not in self.error_handlers:
48-
error_stack.append(stack.pop())
49-
50-
# Second, fetch tokens until error is isolated
51-
if len(stack) == 0:
52-
expect_toks = ",".join(self.parse_table[cur_state].keys())
53-
raise ParsionParseError(f'Unexpected {tok_name}, expected {expect_toks}')
54-
55-
state_error_handlers = self.error_handlers[stack[-1][1]]
56-
57-
error_tokens = []
58-
while len(tokens) > 0 and tokens[0][0] not in state_error_handlers:
59-
error_tokens.append(tokens.pop(0))
60-
61-
if len(tokens) == 0:
61+
try:
62+
# First, pop stack until error handler
63+
error_stack = []
64+
while stack[-1][1] not in self.error_handlers:
65+
error_stack.append(stack.pop())
66+
67+
error_handlers = self.error_handlers[stack[-1][1]]
68+
69+
error_tokens = []
70+
while tokens[0][0] not in error_handlers:
71+
error_tokens.append(tokens.pop(0))
72+
73+
# Call error handler, mimic a reduce operation
74+
error_gen, error_handler = error_handlers[tokens[0][0]]
75+
value = self._call_error_handler(
76+
error_handler,
77+
error_stack,
78+
error_tokens
79+
)
80+
tokens.insert(0, (error_gen, value))
81+
except IndexError:
6282
expect_toks = ",".join(self.parse_table[cur_state].keys())
63-
raise ParsionParseError(f'Unexpected {tok_name}, expected {expect_toks}')
64-
65-
# Call error handler, mimic a reduce operation
66-
error_gen, error_handler = state_error_handlers[tokens[0][0]]
67-
value = self._call_error_handler(
68-
error_handler,
69-
error_stack,
70-
error_tokens
71-
)
72-
tokens.insert(0, (error_gen, value))
83+
raise ParsionParseError(
84+
f'Unexpected {tok_name}, expected {expect_toks}')
7385
continue
74-
86+
7587
op, id = self.parse_table[cur_state][tok_name]
7688
if op == 's':
7789
# shift
@@ -86,31 +98,32 @@ def parse(self, input):
8698
))
8799
stack = stack[:-len(accepts)]
88100
else:
89-
raise ParsionInternalError('Internal error: neigher shift nor reduce')
90-
101+
raise ParsionInternalError(
102+
'Internal error: neigher shift nor reduce')
103+
91104
# Stack contains three elements:
92105
# 0. ('START', ...) - bootstrap
93106
# 1. ('entry', ...) - excpeted result
94107
# 2. ('END', ...) - terminination
95108
# Therefore, pick out entry value and return
96109
return stack[1][0]
97110

98-
def print(self):
111+
def print(self): # pragma: no cover
99112
from tabulate import tabulate
100-
113+
101114
def _print_header(header):
102115
print("")
103116
print(f"{header}")
104-
117+
105118
_print_header("Lexer")
106119
print(tabulate(
107120
[
108121
(name, regexp.pattern)
109-
for name, regexp, handler in self.lex.rules
122+
for name, regexp, handler in self.lex.rules
110123
],
111124
tablefmt='simple_outline'
112125
))
113-
126+
114127
_print_header("FSM grammar")
115128
print(tabulate(
116129
self.parse_grammar,
@@ -124,7 +137,7 @@ def _print_header(header):
124137
[
125138
{
126139
k: " ".join(str(p) if p is not None else '-' for p in v)
127-
for k,v in st.items()
140+
for k, v in st.items()
128141
}
129142
for st in self.parse_table
130143
],
@@ -134,4 +147,4 @@ def _print_header(header):
134147
))
135148

136149
def entry(self, v):
137-
return v#
150+
return v

‎parsion/exceptions.py‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,8 +1,14 @@
11
class ParsionException(Exception):
22
pass
33

4-
class ParsionInternalError(Exception):
4+
5+
class ParsionGeneratorError(ParsionException):
56
pass
67

8+
9+
class ParsionInternalError(ParsionException):
10+
pass
11+
12+
713
class ParsionSelfCheckError(ParsionException):
814
pass

0 commit comments

Comments
 (0)