Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit b9e2cd2

Browse files
juanjuxjuanjux
authored
Merge pull request #88 from juanjux/feature/simplified_astimprover
More simplified AstImprover
2 parents d9724ed + d6fb915 commit b9e2cd2

File tree

89 files changed

+764
-1541
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

89 files changed

+764
-1541
lines changed

‎ANNOTATION.md‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -36,10 +36,9 @@
3636
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Invert'\] | OpBitwiseComplement |
3737
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='UAdd'\] | OpPositive |
3838
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='USub'\] | OpNegative |
39-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='StringLiteral'\] | StringLiteral, Expression |
40-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='ByteLiteral'\] | ByteStringLiteral, Expression |
41-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='NumLiteral'\] | NumberLiteral, Expression |
4239
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Str'\] | StringLiteral, Expression |
40+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Bytes'\] | ByteStringLiteral, Expression |
41+
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Num'\] | NumberLiteral, Expression |
4342
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='BoolLiteral'\] | BooleanLiteral, Expression |
4443
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='JoinedStr'\] | StringLiteral, Expression |
4544
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='JoinedStr'\]/\*\[@InternalType='FormattedValue'\] | Expression, Incomplete |
@@ -150,7 +149,6 @@
150149
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='While'\]/\*\[@internalRole\]\[@internalRole='test'\] | WhileCondition |
151150
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='While'\]/\*\[@InternalType='While\.orelse'\] | IfElse |
152151
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Pass'\] | Noop, Statement |
153-
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Num'\] | NumberLiteral, Expression |
154152
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Assert'\] | Assert, Statement |
155153
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Exec'\] | Call, Expression |
156154
| /self::\*\[@InternalType='Module'\]//\*\[@InternalType='Exec'\]/\*\[@internalRole\]\[@internalRole='body'\] | CallPositionalArgument |

‎driver/normalizer/annotation.go‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -88,10 +88,9 @@ var AnnotationRules = On(Any).Self(
8888
On(HasInternalType(pyast.UAdd)).Roles(OpPositive),
8989
On(HasInternalType(pyast.USub)).Roles(OpNegative),
9090

91-
On(HasInternalType(pyast.StringLiteral)).Roles(StringLiteral, Expression),
92-
On(HasInternalType(pyast.ByteLiteral)).Roles(ByteStringLiteral, Expression),
93-
On(HasInternalType(pyast.NumLiteral)).Roles(NumberLiteral, Expression),
9491
On(HasInternalType(pyast.Str)).Roles(StringLiteral, Expression),
92+
On(HasInternalType(pyast.Bytes)).Roles(ByteStringLiteral, Expression),
93+
On(HasInternalType(pyast.Num)).Roles(NumberLiteral, Expression),
9594
On(HasInternalType(pyast.BoolLiteral)).Roles(BooleanLiteral, Expression),
9695
On(HasInternalType(pyast.JoinedStr)).Roles(StringLiteral, Expression).Children(
9796
On(HasInternalType(pyast.FormattedValue)).Roles(Expression, Incomplete),
@@ -289,7 +288,6 @@ var AnnotationRules = On(Any).Self(
289288
On(HasInternalType("While.orelse")).Roles(IfElse),
290289
),
291290
On(HasInternalType(pyast.Pass)).Roles(Noop, Statement),
292-
On(HasInternalType(pyast.Num)).Roles(NumberLiteral, Expression),
293291
// FIXME: this is the annotated assignment (a: annotation = 3) not exactly Assignment
294292
// it also lacks AssignmentValue and AssignmentVariable (see how to add them)
295293
On(HasInternalType(pyast.Assert)).Roles(Assert, Statement),

‎driver/normalizer/parser.go‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,9 @@ var ToNoder = &native.ObjectToNoder{
1919
"id": true, // Name nodes
2020
"attr": true, // something.attr
2121
"arg": true, // function arguments
22-
"LiteralValue": true, // string/num/byte/constant literal
22+
"LiteralValue": true, // boolean/None literal
23+
"s": true, // string/byte
24+
"n": true, // numeric literal
2325
"noop_line": true, // Comment/Noop (non significative whitespace)
2426
},
2527
SyntheticTokens: map[string]string{

‎driver/normalizer/pyast/pyast.go‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,6 @@ const (
3737
BoolOp = "BoolOp"
3838
BoolopInternal = "boolop"
3939
Break = "Break"
40-
ByteLiteral = "ByteLiteral"
4140
Bytes = "Bytes"
4241
Call = "Call"
4342
ClassDef = "ClassDef"
@@ -103,7 +102,6 @@ const (
103102
NotEq = "NotEq"
104103
NotIn = "NotIn"
105104
Num = "Num"
106-
NumLiteral = "NumLiteral"
107105
Operator = "operator"
108106
Or = "Or"
109107
Param = "Param"
@@ -126,7 +124,6 @@ const (
126124
Stmt = "stmt"
127125
Store = "Store"
128126
Str = "Str"
129-
StringLiteral = "StringLiteral"
130127
Sub = "Sub"
131128
Subscript = "Subscript"
132129
Suite = "Suite"

‎native/python_package/python_driver/astimprove.py‎

Lines changed: 62 additions & 92 deletions
Original file line numberDiff line numberDiff line change
@@ -193,6 +193,65 @@ def _create_astmissing_lines(self):
193193
self._current_line = len(lines)
194194
return lines
195195

196+
def add_noops(self, node, visit_dict, root):
197+
if not isinstance(visit_dict, dict):
198+
return visit_dict
199+
200+
def _create_nooplines_list(startline, noops_previous):
201+
nooplines = []
202+
curline = startline
203+
for noopline in noops_previous:
204+
nooplines.append({
205+
"ast_type": "NoopLine",
206+
"noop_line": noopline,
207+
"lineno": curline,
208+
"col_offset": 1,
209+
})
210+
curline += 1
211+
return nooplines
212+
213+
# Add all the noop (whitespace and comments) lines between the
214+
# last node and this one
215+
noops_previous, startline, endline, endcol =\
216+
self.previous_nooplines(node)
217+
if noops_previous:
218+
visit_dict['noops_previous'] = {
219+
"ast_type": "PreviousNoops",
220+
"lineno": startline,
221+
"col_offset": 1,
222+
"end_lineno": endline,
223+
"end_col_offset": max(endcol, 1),
224+
"lines": _create_nooplines_list(startline, noops_previous)
225+
}
226+
227+
# Other noops at the end of its significative line except the implicit
228+
# finishing newline
229+
noops_sameline = self.sameline_remainder_noops(node)
230+
joined_sameline = ''.join([x['value'] for x in noops_sameline])
231+
if noops_sameline:
232+
visit_dict['noops_sameline'] = {
233+
"ast_type": "SameLineNoops",
234+
"lineno": node.get("lineno", 0),
235+
"col_offset": noops_sameline[0]["colstart"],
236+
"noop_line": joined_sameline,
237+
"end_lineno": node.get("lineno", 0),
238+
"end_col_offset": max(noops_sameline[-1]["colend"], 1)
239+
}
240+
241+
# Finally, if this is the root node, add all noops after the last op node
242+
if root:
243+
noops_remainder, startline, endline, endcol =\
244+
self.remainder_noops()
245+
if noops_remainder:
246+
visit_dict['noops_remainder'] = {
247+
"ast_type": "RemainderNoops",
248+
"lineno": startline,
249+
"col_offset": 1,
250+
"end_lineno": endline,
251+
"end_col_offset": max(endcol, 1),
252+
"lines": _create_nooplines_list(startline, noops_remainder)
253+
}
254+
196255
def previous_nooplines(self, nodedict):
197256
"""Return a list of the preceding comment and blank lines"""
198257
previous = []
@@ -286,7 +345,7 @@ def remainder_noops(self):
286345

287346

288347
_TOKEN_KEYS = set(
289-
("module", "name", "id", "attr", "arg", "LiteralValue")
348+
("module", "name", "id", "attr", "arg", "LiteralValue", "s", "n")
290349
)
291350

292351
_SYNTHETIC_TOKENS = {
@@ -360,65 +419,6 @@ def __init__(self, codestr, astdict):
360419

361420
self.visit_Global = self.visit_Nonlocal = self._promote_names
362421

363-
def _add_noops(self, node, visit_dict, root):
364-
if not isinstance(visit_dict, dict):
365-
return visit_dict
366-
367-
def _create_nooplines_list(startline, noops_previous):
368-
nooplines = []
369-
curline = startline
370-
for noopline in noops_previous:
371-
nooplines.append({
372-
"ast_type": "NoopLine",
373-
"noop_line": noopline,
374-
"lineno": curline,
375-
"col_offset": 1,
376-
})
377-
curline += 1
378-
return nooplines
379-
380-
# Add all the noop (whitespace and comments) lines between the
381-
# last node and this one
382-
noops_previous, startline, endline, endcol =\
383-
self.noops_sync.previous_nooplines(node)
384-
if noops_previous:
385-
visit_dict['noops_previous'] = {
386-
"ast_type": "PreviousNoops",
387-
"lineno": startline,
388-
"col_offset": 1,
389-
"end_lineno": endline,
390-
"end_col_offset": max(endcol, 1),
391-
"lines": _create_nooplines_list(startline, noops_previous)
392-
}
393-
394-
# Other noops at the end of its significative line except the implicit
395-
# finishing newline
396-
noops_sameline = self.noops_sync.sameline_remainder_noops(node)
397-
joined_sameline = ''.join([x['value'] for x in noops_sameline])
398-
if noops_sameline:
399-
visit_dict['noops_sameline'] = {
400-
"ast_type": "SameLineNoops",
401-
"lineno": node.get("lineno", 0),
402-
"col_offset": noops_sameline[0]["colstart"],
403-
"noop_line": joined_sameline,
404-
"end_lineno": node.get("lineno", 0),
405-
"end_col_offset": max(noops_sameline[-1]["colend"], 1)
406-
}
407-
408-
# Finally, if this is the root node, add all noops after the last op node
409-
if root:
410-
noops_remainder, startline, endline, endcol =\
411-
self.noops_sync.remainder_noops()
412-
if noops_remainder:
413-
visit_dict['noops_remainder'] = {
414-
"ast_type": "RemainderNoops",
415-
"lineno": startline,
416-
"col_offset": 1,
417-
"end_lineno": endline,
418-
"end_col_offset": max(endcol, 1),
419-
"lines": _create_nooplines_list(startline, noops_remainder)
420-
}
421-
422422
def parse(self):
423423
res = self.visit(self._astdict, root=True)
424424
return res
@@ -434,7 +434,7 @@ def visit(self, node, root=False):
434434

435435
meth = getattr(self, "visit_" + node_type, self.visit_other)
436436
visit_result = meth(node)
437-
self._add_noops(node, visit_result, root)
437+
self.noops_sync.add_noops(node, visit_result, root)
438438
self.pos_sync.sync_node_pos(visit_result)
439439

440440
if not self.codestr:
@@ -461,11 +461,6 @@ def visit_str(self, node):
461461
"""
462462
return str(node)
463463

464-
def visit_Str(self, node):
465-
node.update({"LiteralValue": node["s"], "ast_type": "StringLiteral"})
466-
node.pop("s", None)
467-
return node
468-
469464
def visit_Bytes(self, node):
470465
try:
471466
s = node["s"].decode()
@@ -475,15 +470,7 @@ def visit_Bytes(self, node):
475470
s = encode(node["s"], 'base64').decode().strip()
476471
encoding = 'base64'
477472

478-
node.update({"LiteralValue": s,
479-
"encoding": encoding,
480-
"ast_type": "ByteLiteral"})
481-
node.pop("s", None)
482-
return node
483-
484-
def visit_NoneType(self, node):
485-
node.update({"LiteralValue": "None",
486-
"ast_type": "NoneLiteral"})
473+
node.update({"s": s, "encoding": encoding})
487474
return node
488475

489476
def _promote_names(self, node):
@@ -512,23 +499,6 @@ def visit_NameConstant(self, node):
512499
node["ast_type"] = "NameConstant"
513500
return node
514501

515-
def visit_Num(self, node):
516-
if isinstance(node["n"], int):
517-
ret_dict = { "NumType": "int", "LiteralValue": node["n"] }
518-
elif isinstance(node["n"], float):
519-
ret_dict = { "NumType": "float", "LiteralValue": node["n"] }
520-
elif isinstance(node["n"], complex):
521-
ret_dict = {
522-
"NumType": "complex",
523-
"LiteralValue": {"real": node["n"]["real"],
524-
"imaginary": node["n"]["imag"]},
525-
}
526-
527-
node["ast_type"] = "NumLiteral"
528-
node.update(ret_dict)
529-
node.pop("n", None)
530-
return node
531-
532502
def visit_other(self, node):
533503
for field in node.get("_fields", []):
534504
meth = getattr(self, "visit_" + node["ast_type"], self.visit_other_field)

‎tests/annotations.py.native‎

Lines changed: 6 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -29,13 +29,12 @@
2929
"lineno": 1
3030
},
3131
"value": {
32-
"LiteralValue": 1,
33-
"NumType": "int",
34-
"ast_type": "NumLiteral",
32+
"ast_type": "Num",
3533
"col_offset": 10,
3634
"end_col_offset": 10,
3735
"end_lineno": 1,
38-
"lineno": 1
36+
"lineno": 1,
37+
"n": 1
3938
}
4039
},
4140
{
@@ -117,13 +116,12 @@
117116
"end_lineno": 4,
118117
"lineno": 4,
119118
"value": {
120-
"LiteralValue": 0,
121-
"NumType": "float",
122-
"ast_type": "NumLiteral",
119+
"ast_type": "Num",
123120
"col_offset": 12,
124121
"end_col_offset": 14,
125122
"end_lineno": 4,
126-
"lineno": 4
123+
"lineno": 4,
124+
"n": 0
127125
}
128126
}
129127
],

‎tests/annotations.py.uast‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -52,7 +52,7 @@ Module {
5252
. . . . . . internalRole: target
5353
. . . . . }
5454
. . . . }
55-
. . . . 2: NumLiteral {
55+
. . . . 2: Num {
5656
. . . . . Roles: NumberLiteral,Expression
5757
. . . . . TOKEN "1"
5858
. . . . . StartPosition: {
@@ -66,7 +66,6 @@ Module {
6666
. . . . . . Col: 10
6767
. . . . . }
6868
. . . . . Properties: {
69-
. . . . . . NumType: int
7069
. . . . . . internalRole: value
7170
. . . . . }
7271
. . . . }
@@ -244,7 +243,7 @@ Module {
244243
. . . . . . . . Col: 10
245244
. . . . . . . }
246245
. . . . . . . Children: {
247-
. . . . . . . . 0: NumLiteral {
246+
. . . . . . . . 0: Num {
248247
. . . . . . . . . Roles: NumberLiteral,Expression
249248
. . . . . . . . . TOKEN "0"
250249
. . . . . . . . . StartPosition: {
@@ -258,7 +257,6 @@ Module {
258257
. . . . . . . . . . Col: 14
259258
. . . . . . . . . }
260259
. . . . . . . . . Properties: {
261-
. . . . . . . . . . NumType: float
262260
. . . . . . . . . . internalRole: value
263261
. . . . . . . . . }
264262
. . . . . . . . }

0 commit comments

Comments
 (0)