Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit 044d6c6

Browse files
juanjuxjuanjux
committed
Update to last SDK, more synth tokens for operators, annotations
- Synthetic token rules for operators - Escaped % synthetictoken do it doesnt intefere with sprintf - More synthetic tokens for operators - Updated integration tests - Added missing UnaryOp tokens - Readded the aditional synthetic tokens for all operators - Use errors.new - Add a newline above the external imports - List of missing annotation, fixed a couple - Updated SDK. Added list of missing annotations. Fixed a couple things. - Added roles for the internal loops and tests of comprehensions - Added roles for the IfCondition children, imperfect until the SDK gets some list mixing features - Added roles for the Comprehension.Condition children, imperfect until the SDK gets some list mixing features - Added roles for formattedvalue (expression currently) and a fstring test - comments - More integration tests for fstrings - Fixed old reference to ASTParserBuilder - Ran GoFmt - Fix license and download links - Removed some leftover tests prints - Import fixes
1 parent 955fc5d commit 044d6c6

17 files changed

+1928
-214
lines changed

‎driver/normalizer/annotation.go‎

Lines changed: 68 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,9 @@ Missing nodes or nodes needing new features from the SDK:
3535
Yield
3636
YieldFrom
3737
38+
= PR 57:
39+
Starred
40+
3841
= PR 58:
3942
withitem
4043
@@ -44,6 +47,11 @@ Missing nodes or nodes needing new features from the SDK:
4447
Slice
4548
ExtSlice
4649
50+
= PR 63:
51+
Lambda
52+
kwarg
53+
FunctionDef.decorator_list
54+
4755
= PR 81:
4856
ListComp
4957
SetComp
@@ -52,35 +60,37 @@ Missing nodes or nodes needing new features from the SDK:
5260
= PR 79:
5361
arguments
5462
55-
=== No PR:
63+
= PR 111:
64+
FormattedValue (InterpolatedValue)
5665
57-
JoinedStr
58-
FormattedValue
66+
= PR 112:
67+
AnnAssign
68+
annotation
5969
60-
Starred
70+
= PR 113:
71+
AsyncFunctionDef (FunctionDef + async)
72+
Await
73+
AsyncFor (For + async)
74+
AsyncWith (With + async)
75+
76+
= PR 114:
77+
Global
78+
Nonlocal
79+
80+
=== No PR:
6181
6282
BoolOp collapsing: needs SDK features
6383
arguments.defaults: needs SDK features
6484
arguments.keywords: same
6585
66-
GeneratorExp: check
67-
68-
AnnAssign (currently as assign, needs Annotation UAST)
69-
70-
Lambda
71-
72-
Global
73-
Nonlocal
74-
75-
AsyncFunctionDef (FunctionDef + async)
76-
Await
77-
AsyncFor (For + async)
78-
AsyncWith (With + async)
79-
86+
These also need SDK list-mix features:
87+
Compare.comparators
88+
Compare.ops
89+
IfCondition.left
90+
(see: https://greentreesnakes.readthedocs.io/en/latest/nodes.html#Compare)
8091
8192
*/
8293

83-
// TODO: add the "stride", third element of slices in some way once we've the index/slice roles
8494
var AnnotationRules = On(Any).Self(
8595
On(Not(HasInternalType(pyast.Module))).Error(errors.New("root must be Module")),
8696
On(HasInternalType(pyast.Module)).Roles(File).Descendants(
@@ -138,9 +148,10 @@ var AnnotationRules = On(Any).Self(
138148
On(HasInternalType(pyast.NumLiteral)).Roles(NumberLiteral),
139149
On(HasInternalType(pyast.Str)).Roles(StringLiteral),
140150
On(HasInternalType(pyast.BoolLiteral)).Roles(BooleanLiteral),
141-
// FIXME: JoinedStr are the fstrings (f"my name is {name}"), they have a composite AST
142-
// with a body that is a list of StringLiteral + FormattedValue(value, conversion, format_spec)
143-
On(HasInternalType(pyast.JoinedStr)).Roles(StringLiteral),
151+
On(HasInternalType(pyast.JoinedStr)).Roles(StringLiteral).Children(
152+
// FIXME: should be StringInterpolatedExpression or something like that
153+
On(HasInternalType(pyast.FormattedValue)).Roles(Expression),
154+
),
144155
On(HasInternalType(pyast.NoneLiteral)).Roles(NullLiteral),
145156
On(HasInternalType(pyast.Set)).Roles(SetLiteral),
146157
On(HasInternalType(pyast.List)).Roles(ListLiteral),
@@ -225,7 +236,7 @@ var AnnotationRules = On(Any).Self(
225236
On(HasInternalRole("handlers")).Roles(TryCatch),
226237
On(HasInternalRole("orelse")).Roles(IfElse),
227238
),
228-
On(HasInternalType(pyast.TryExcept)).Roles(TryCatch), // py2
239+
On(HasInternalType(pyast.TryExcept)).Roles(TryCatch), // py2
229240
On(HasInternalType(pyast.ExceptHandler)).Roles(TryCatch), // py3
230241
On(HasInternalType(pyast.TryFinally)).Roles(TryFinally),
231242
On(HasInternalType(pyast.Raise)).Roles(Throw),
@@ -236,10 +247,27 @@ var AnnotationRules = On(Any).Self(
236247
On(HasInternalType(pyast.Return)).Roles(Return),
237248
On(HasInternalType(pyast.Break)).Roles(Break),
238249
On(HasInternalType(pyast.Continue)).Roles(Continue),
250+
// FIXME: IfCondition bodies in Python take the form:
251+
// 1 < a < 10
252+
// - left (internalRole): 1 (first element)
253+
// - Compare.ops (internalType): [LessThan, LessThan]
254+
// - Compare.comparators (internalType): ['a', 10]
255+
// The current mapping is:
256+
// - left: BinaryExpressionLeft
257+
// - Compare.ops: BinaryExpressionOp
258+
// - Compare.comparators: BinaryExpressionRight
259+
// But this is obviously not correct. To fix this properly we would need
260+
// and SDK feature to mix lists (also needed for default and keyword arguments and
261+
// boolean operators).
262+
// "If that sounds awkward is because it is" (their words)
239263
On(HasInternalType(pyast.If)).Roles(If).Children(
240264
On(HasInternalType("If.body")).Roles(IfBody),
241-
On(HasInternalType(pyast.Compare)).Roles(IfCondition),
242265
On(HasInternalType("If.orelse")).Roles(IfElse),
266+
On(HasInternalType(pyast.Compare)).Roles(IfCondition, BinaryExpression).Children(
267+
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
268+
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
269+
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
270+
),
243271
),
244272
On(HasInternalType(pyast.IfExp)).Roles(If, Expression).Children(
245273
// These are used on ifexpressions (a = 1 if x else 2)
@@ -296,11 +324,27 @@ var AnnotationRules = On(Any).Self(
296324
// information by themselves and this we consider it comments (some preprocessors or linters can use
297325
// them, the runtimes ignore them). The TOKEN will take the annotation in the UAST node so
298326
// the information is keept in any case.
327+
// FIXME: change to Annotation when PR 112 is merged
299328
On(HasInternalRole("annotation")).Roles(Comment),
300329
On(HasInternalRole("returns")).Roles(Comment),
301330

302331
// Python very odd ellipsis operator. Has a special rule in tonoder synthetic tokens
303332
// map to load it with the token "PythonEllipsisOperator" and gets the role SimpleIdentifier
304333
On(HasInternalType(pyast.Ellipsis)).Roles(SimpleIdentifier),
334+
335+
// List/Map/Set comprehensions. We map the "for x in y" to ForEach roles and the
336+
// "if something" to If* roles. FIXME: missing the top comprehension roles in the UAST, change
337+
// once they've been merged
338+
On(HasInternalType(pyast.Comprehension)).Roles(ForEach).Children(
339+
On(HasInternalRole("iter")).Roles(ForUpdate),
340+
On(HasInternalRole("target")).Roles(ForExpression),
341+
// FIXME: see the comment on IfCondition above
342+
On(HasInternalType(pyast.Compare)).Roles(IfCondition, BinaryExpression).Children(
343+
On(HasInternalType("Compare.ops")).Roles(BinaryExpressionOp),
344+
On(HasInternalType("Compare.comparators")).Roles(BinaryExpressionRight),
345+
On(HasInternalRole("left")).Roles(BinaryExpressionLeft),
346+
),
347+
),
348+
305349
),
306350
)

‎driver/normalizer/annotation_test.go‎

Lines changed: 0 additions & 11 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,7 @@
11
package normalizer
22

33
import (
4-
"bytes"
54
"encoding/json"
6-
"fmt"
75
"os"
86
"path/filepath"
97
"testing"
@@ -53,11 +51,6 @@ func TestAnnotatePrettyAnnotationsOnly(t *testing.T) {
5351

5452
err = AnnotationRules.Apply(n)
5553
require.NoError(err)
56-
57-
//buf := bytes.NewBuffer(nil)
58-
//err = uast.Pretty(n, buf, uast.IncludeAnnotations|uast.IncludeChildren|uast.IncludeTokens)
59-
//require.NoError(err)
60-
//fmt.Println(buf.String())
6154
}
6255

6356
func TestNodeTokens(t *testing.T) {
@@ -73,10 +66,8 @@ func TestNodeTokens(t *testing.T) {
7366
tokens := uast.Tokens(n)
7467
require.True(len(tokens) > 0)
7568
//func Pretty(n *Node, w io.Writer, includes IncludeFlag) error {
76-
buf := bytes.NewBuffer(nil)
7769
err = uast.Pretty(n, os.Stdout, uast.IncludeAll)
7870
require.NoError(err)
79-
fmt.Println(buf.String())
8071
}
8172

8273
func TestAll(t *testing.T) {
@@ -96,10 +87,8 @@ func TestAll(t *testing.T) {
9687
require.NoError(err)
9788

9889
//func Pretty(n *Node, w io.Writer, includes IncludeFlag) error {
99-
buf := bytes.NewBuffer(nil)
10090
err = uast.Pretty(n, os.Stdout, uast.IncludeAll)
10191
require.NoError(err)
102-
fmt.Println(buf.String())
10392
}
10493

10594
func getFixture(name string) (map[string]interface{}, error) {

‎driver/normalizer/parser.go‎

Lines changed: 20 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -31,25 +31,25 @@ var ToNoder = &native.ObjectToNoder{
3131
"Mod": "%%",
3232
"Pow": "**",
3333
"AugAssign": "?=",
34-
"BitAnd" : "&",
35-
"BitOr" : "|",
36-
"BitXor" : "^",
37-
"LShift" : "<<",
38-
"RShift" : ">>",
39-
"Eq" : "==",
40-
"NotEq" : "!=",
41-
"Not" : "!",
42-
"Lt" : "<",
43-
"LtE" : "<=",
44-
"Gt" : ">",
45-
"GtE" : ">=",
46-
"Is" : "is",
47-
"IsNot" : "not is",
48-
"In" : "in",
49-
"NotIn" : "not in",
50-
"UAdd" : "+",
51-
"USub" : "-",
52-
"Invert" : "~",
34+
"BitAnd": "&",
35+
"BitOr": "|",
36+
"BitXor": "^",
37+
"LShift": "<<",
38+
"RShift": ">>",
39+
"Eq": "==",
40+
"NotEq": "!=",
41+
"Not": "!",
42+
"Lt": "<",
43+
"LtE": "<=",
44+
"Gt": ">",
45+
"GtE": ">=",
46+
"Is": "is",
47+
"IsNot": "not is",
48+
"In": "in",
49+
"NotIn": "not in",
50+
"UAdd": "+",
51+
"USub": "-",
52+
"Invert": "~",
5353
},
5454
PromoteAllPropertyLists: false,
5555
PromotedPropertyLists: map[string]map[string]bool{
@@ -75,7 +75,7 @@ var ToNoder = &native.ObjectToNoder{
7575
// FIXME: test[ast_type=Compare].comparators is a list?? (should be "right")
7676
}
7777

78-
// UASTParserBuilder creates a parser that transform source code files into *uast.Node.
78+
// ASTParserBuilder creates a parser that transform source code files into *uast.Node.
7979
func UASTParserBuilder(opts driver.UASTParserOptions) (driver.UASTParser, error) {
8080
parser, err := native.ExecParser(ToNoder, opts.NativeBin)
8181
if err != nil {

‎driver/normalizer/parser_test.go‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,6 @@
11
package normalizer
22

33
import (
4-
"fmt"
54
"testing"
65

76
"github.com/stretchr/testify/require"
@@ -16,5 +15,4 @@ func TestNativeToNoder(t *testing.T) {
1615
n, err := ToNoder.ToNode(f)
1716
require.NoError(err)
1817
require.NotNil(n)
19-
fmt.Println("NODE", n)
2018
}

0 commit comments

Comments
 (0)