Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit 6d65ef5

Browse files
author
Juanjo Alvarez
committed
Splitted the astimprover megamodule into several + many fixes and improvements
- Normalized function arguments, updated annotations to the new format - Removed positions since they don't work when endpos is missing. Added new argument annotations - Remove two now unneeded workarounds (fix in Python ast module and fix in the SDK) - Use uast:Identifiers instead of str for Name fields in semantic objects - FunctionType.Arguments should be an Array - Un-wtfized python's qualified identifiers. Updated SDK. Regenerated tests - Update fixtures - Map normal arg as uast.Argument - Changes to run go test locally - Changes some files to "go test" can be run only running `make native-build`. Unfortunately, this breaks `make test` and `make integration-test`. WIP. - Add missing file - Correct the column of QualifiedIdentifier children - Fix comments - Fix make test Signed-off-by: Juanjo Alvarez <[email protected]> rebase this Signed-off-by: Juanjo Alvarez <[email protected]>
1 parent 2dea82e commit 6d65ef5

File tree

322 files changed

+171488
-103415
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

322 files changed

+171488
-103415
lines changed

‎.gitignore‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,4 +5,4 @@ build
55
.cache/*
66
__pycache__
77
makebuild.sh
8-
.vscode/launch.json
8+
fixtures/*_got

‎Makefile‎

Lines changed: 7 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -1,15 +1,17 @@
11
DEV_DEPS ?= native/dev_deps
2+
PIP2_CMD ?= pip2 install -U --prefix=$(BUILD_PATH)/.local
3+
PIP3_CMD ?= pip3 install -U --prefix=$(BUILD_PATH)/.local
24

35
test-native-internal:
4-
pip3 install --user ${DEV_DEPS}/python-pydetector/ || pip3 install --user pydetector-bblfsh
5-
pip2 install --user ${DEV_DEPS}/python-pydetector/ || pip2 install --user pydetector-bblfsh
6+
$(PIP2_CMD) ${DEV_DEPS}/python-pydetector/ || $(PIP2_CMD) pydetector-bblfsh
7+
$(PIP3_CMD) ${DEV_DEPS}/python-pydetector/ || $(PIP3_CMD) pydetector-bblfsh
68
cd native/python_package/test && \
79
python3 -m unittest discover
810

911
build-native-internal:
10-
pip3 install --user ${DEV_DEPS}/python-pydetector/ || pip3 install --user pydetector-bblfsh
11-
pip2 install --user ${DEV_DEPS}/python-pydetector/ || pip2 install --user pydetector-bblfsh
12-
cd native/python_package/ && pip3 install -U --user .
12+
$(PIP3_CMD) ${DEV_DEPS}/python-pydetector/ || $(PIP3_CMD) pydetector-bblfsh
13+
$(PIP2_CMD) ${DEV_DEPS}/python-pydetector/ || $(PIP2_CMD) pydetector-bblfsh
14+
cd native/python_package/ && $(PIP3_CMD) .
1315
cp native/sh/native.sh $(BUILD_PATH)/bin/native;
1416
chmod +x $(BUILD_PATH)/bin/native
1517

‎driver/fixtures/fixtures_test.go‎

Lines changed: 33 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@ import (
77
"github.com/bblfsh/python-driver/driver/normalizer"
88
"gopkg.in/bblfsh/sdk.v2/sdk/driver"
99
"gopkg.in/bblfsh/sdk.v2/sdk/driver/fixtures"
10+
"os"
1011
)
1112

1213
const projectRoot = "../../"
@@ -16,34 +17,46 @@ var Suite = &fixtures.Suite{
1617
Ext: ".py",
1718
Path: filepath.Join(projectRoot, fixtures.Dir),
1819
NewDriver: func() driver.BaseDriver {
19-
return driver.NewExecDriverAt(filepath.Join(projectRoot, "build/bin/native"))
20+
return driver.NewExecDriverAt(filepath.Join(projectRoot,
21+
"build/.local/bin/python_driver"))
2022
},
21-
Transforms: driver.Transforms{
22-
Preprocess: normalizer.Preprocess,
23-
Normalize: normalizer.Normalize,
24-
Native: normalizer.Native,
25-
Code: normalizer.Code,
26-
},
27-
BenchName: "issue_server101",
28-
// XXX check this
29-
//Semantic: fixtures.SemanticConfig{
30-
//BlacklistTypes: []string{
31-
//"StringLiteral",
32-
//"SimpleName",
33-
//"QualifiedName",
34-
//"BlockComment",
35-
//"LineComment",
36-
//"Block",
37-
//"ImportDeclaration",
38-
//"MethodDeclaration",
39-
//},
23+
Transforms: normalizer.Transforms,
24+
BenchName: "issue_server101",
25+
//Docker:fixtures.DockerConfig{
26+
//Image:"python:3",
4027
//},
28+
Semantic: fixtures.SemanticConfig{
29+
BlacklistTypes: []string{
30+
"AsyncFunctionDef",
31+
"Bytes",
32+
"FunctionDef",
33+
"Import",
34+
"ImportFrom",
35+
"Name",
36+
"NoopLine",
37+
"NoopSameLine",
38+
"QualifiedIdentifier",
39+
"Str",
40+
"StringLiteral",
41+
"arg",
42+
"kwarg",
43+
"kwonly_arg",
44+
"vararg",
45+
},
46+
},
47+
}
48+
49+
func SetPythonPath() {
50+
os.Setenv("PYTHONPATH", filepath.Join(projectRoot,
51+
"build/.local/lib/python3.6/site-packages"))
4152
}
4253

4354
func TestPythonDriver(t *testing.T) {
55+
SetPythonPath()
4456
Suite.RunTests(t)
4557
}
4658

4759
func BenchmarkPythonDriver(b *testing.B) {
60+
SetPythonPath()
4861
Suite.RunBenchmarks(b)
4962
}

‎driver/main.go‎

Lines changed: 1 addition & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -8,10 +8,5 @@ import (
88
)
99

1010
func main() {
11-
driver.Run(driver.Transforms{
12-
Preprocess: normalizer.Preprocess,
13-
Normalize: normalizer.Normalize,
14-
Native: normalizer.Native,
15-
Code: normalizer.Code,
16-
})
11+
driver.Run(normalizer.Transforms)
1712
}

‎driver/normalizer/annotation.go‎

Lines changed: 21 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -145,6 +145,7 @@ func (op opLevelDotsNumConv) Construct(st *State, n nodes.Node) (nodes.Node, err
145145
var _ Op = opLevelDotsNumConv{}
146146

147147
var Annotations = []Mapping{
148+
// FIXME: doesnt work
148149
AnnotateType("Module", nil, role.File, role.Module),
149150

150151
// Comparison operators
@@ -233,6 +234,7 @@ var Annotations = []Mapping{
233234
role.Identifier, role.Expression),
234235
AnnotateType("Attribute", FieldRoles{"attr": {Rename: uast.KeyToken}},
235236
role.Identifier, role.Expression),
237+
AnnotateType("QualifiedIdentifier", nil, role.Identifier, role.Expression, role.Qualified),
236238

237239
// Binary Expressions
238240
AnnotateType("BinOp", ObjRoles{
@@ -370,29 +372,21 @@ var Annotations = []Mapping{
370372
}), role.Function, role.Declaration, role.Value, role.Anonymous),
371373

372374
// Formal Arguments
373-
// FIXME: opt: true + arr: true seems to cause a crash in the SDK
374-
AnnotateType("arguments", FieldRoles{
375-
"args": {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.Argument, role.Name, role.Identifier}},
376-
"defaults": {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Value, role.Default}},
377-
// Default arguments: Python's AST puts default arguments on a sibling list to the one of
378-
// arguments that must be mapped to the arguments right-aligned like:
379-
// a, b=2, c=3 ->
380-
// args [a,b,c],
381-
// defaults [2,3]
382-
"kwarg": {Opt: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Map, role.Name, role.Identifier}},
383-
"vararg": {Opt: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Name, role.Identifier}},
384-
// forced keyword parameters (def kwonly(*, a=1, b=2))
385-
"kwonlyargs": {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Name, role.Identifier, role.Incomplete}},
386-
// the defaults of the above, same right-align matching
387-
"kw_defaults": {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Map, role.Value, role.Default}},
388-
}, role.Function, role.Declaration, role.Argument, role.Incomplete),
389-
390-
AnnotateType("arguments", FieldRoles{
391-
"args": {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.Argument, role.Name, role.Identifier}},
392-
"defaults": {Arr: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Value, role.Default}},
393-
"kwarg": {Opt: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Map, role.Name, role.Identifier}},
394-
"vararg": {Opt: true, Roles: role.Roles{role.Function, role.Declaration, role.ArgsList, role.Name, role.Identifier}},
395-
}, role.Function, role.Declaration, role.Argument, role.Incomplete),
375+
// Incomplete because we've no role for "virtual" grouping nodes
376+
AnnotateType("arguments", nil, role.Function, role.Declaration, role.Argument, role.Incomplete),
377+
AnnotateType("arg",
378+
FieldRoles{
379+
"default": {Opt: true, Roles: role.Roles{role.Argument, role.Default}},
380+
"annotation": {Opt: true, Roles: role.Roles{role.Annotation, role.Noop}},
381+
}, role.Function, role.Declaration, role.Argument, role.Name),
382+
// Incomplete because we've no role for "mandatory name"
383+
AnnotateType("kwonly_arg",
384+
FieldRoles{
385+
"default": {Opt: true, Roles: role.Roles{role.Argument, role.Default}},
386+
"annotation": {Opt: true, Roles: role.Roles{role.Annotation, role.Noop}},
387+
}, role.Function, role.Declaration, role.Argument, role.Name, role.Incomplete),
388+
AnnotateType("kwarg", nil, role.Function, role.Declaration, role.ArgsList, role.Map, role.Name),
389+
AnnotateType("vararg", nil, role.Function, role.Declaration, role.ArgsList, role.List, role.Name),
396390

397391
// Function Calls
398392
AnnotateType("Call", FieldRoles{
@@ -427,10 +421,11 @@ var Annotations = []Mapping{
427421
}, role.Noop, role.Comment),
428422

429423
// Qualified Identifiers
424+
// FIXME XXX remove
430425
// a.b.c ("a" and "b" will be Qualified+Identifier, "c" will be just Identifier)
431-
AnnotateType("Attribute", FieldRoles{
432-
"value": {Arr: true, Roles: role.Roles{role.Qualified}},
433-
}),
426+
//AnnotateType("Attribute", FieldRoles{
427+
// "value": {Opt: true, Roles: role.Roles{role.Qualified}},
428+
//}),
434429

435430
// Import
436431
AnnotateType("Import", nil, role.Import, role.Declaration, role.Statement),

‎driver/normalizer/normalizer.go‎

Lines changed: 83 additions & 55 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,6 @@ package normalizer
22

33
import (
44
"gopkg.in/bblfsh/sdk.v2/uast"
5-
"gopkg.in/bblfsh/sdk.v2/uast/nodes"
65
. "gopkg.in/bblfsh/sdk.v2/uast/transformer"
76
)
87

@@ -13,67 +12,26 @@ var Preprocess = Transformers([][]Transformer{
1312
var Preprocessors = []Mapping{
1413
ObjectToNode{
1514
InternalTypeKey: "ast_type",
16-
// FIXME: restore once positions are working again
17-
// LineKey: "lineno",
18-
// ColumnKey: "col_offset",
19-
// EndLineKey: "end_lineno",
20-
// EndColumnKey: "end_col_offset",
15+
LineKey: "lineno",
16+
ColumnKey: "col_offset",
17+
EndLineKey: "end_lineno",
18+
EndColumnKey: "end_col_offset",
2119
}.Mapping(),
2220
}
2321

2422
var Normalize = Transformers([][]Transformer{
2523
{Mappings(Normalizers...)},
2624
}...)
2725

28-
type opFuncArguments struct {
29-
args Op
30-
}
31-
32-
func (op opFuncArguments) Kinds() nodes.Kind {
33-
return nodes.KindArray
34-
}
35-
36-
func (op opFuncArguments) Check(st *State, n nodes.Node) (bool, error) {
37-
v, ok := n.(nodes.Array)
38-
if !ok {
39-
return false, nil
40-
}
41-
42-
return op.args.Check(st, v)
43-
}
44-
45-
func (op opFuncArguments) Construct(st *State, n nodes.Node) (nodes.Node, error) {
46-
// Iterate over n.args constructing Arguments. Set the arg.Init to the values
47-
// in n.defaults. Do the same for kwonlyargs/kw_defaults. Finally add vararg
48-
// and kwargs if they are set
49-
n, err := op.args.Construct(st, n)
50-
51-
if err != nil {
52-
return nil, err
53-
}
54-
55-
// FIXME: implement
56-
57-
return n, nil
58-
}
59-
6026
// FIXME: decorators? (annotations/tags)
61-
// FIXME: in Python, an argument being variadic or not depends on being on
62-
// args.[kwonlyargs|args] or plain "args" not on any property of the argument
63-
// nodes themselves. Check with @dennys about how to map this.
6427
func funcDefMap(typ string, async bool) Mapping {
6528
return MapSemantic(typ, uast.FunctionGroup{}, MapObj(
6629
Obj{
6730
"body": Var("body"),
68-
"name": Var("name"),
31+
uast.KeyToken: Var("name"),
32+
// Arguments should be converted by the uast.Arguments normalization
6933
"args": Obj{
70-
"args": Each("normal_args", Var("normal_arg_name")),
71-
"kwonlyargs": Each("kw_args",
72-
Obj{
73-
"arg": Var("kwarg_name"),
74-
"value": Var("kwarg_default"),
75-
}),
76-
"varargs": Var("varargs"),
34+
"args": Var("arguments"),
7735
},
7836
},
7937
Obj{
@@ -83,10 +41,12 @@ func funcDefMap(typ string, async bool) Mapping {
8341
"async": Bool(async),
8442
},
8543
UASTType(uast.Alias{}, Obj{
86-
"Name": Var("name"),
44+
"Name": UASTType(uast.Identifier{}, Obj{
45+
"Name": Var("name"),
46+
}),
8747
"Node": UASTType(uast.Function{}, Obj{
8848
"Type": UASTType(uast.FunctionType{}, Obj{
89-
"Arguments": Var("args"),
49+
"Arguments": Var("arguments"),
9050
}),
9151
"Body": Var("body"),
9252
}),
@@ -132,25 +92,93 @@ var Normalizers = []Mapping{
13292
Obj{"Name": Var("name")},
13393
)),
13494

95+
MapSemantic("Name", uast.Identifier{}, MapObj(
96+
Obj{"attr": Var("name")},
97+
Obj{"Name": Var("name")},
98+
)),
99+
135100
// FIXME: check that the identifiers are in the right order
136-
MapSemantic("Attribute", uast.QualifiedIdentifier{}, MapObj(
137-
Obj{"value": Var("identifiers")},
101+
// FIXME: check that those are uast:Identifiers
102+
MapSemantic("QualifiedIdentifier", uast.QualifiedIdentifier{}, MapObj(
103+
Obj{"identifiers": Var("identifiers")},
138104
Obj{"Names": Var("identifiers")},
139105
)),
140106

141107
MapSemantic("NoopLine", uast.Comment{}, MapObj(
142-
Obj{"noop_line": CommentText([2]string{"#", ""}, "comm")},
108+
Obj{"noop_line": CommentText([2]string{}, "comm")},
143109
CommentNode(false, "comm", nil),
144110
)),
145111

146112
MapSemantic("NoopSameLine", uast.Comment{}, MapObj(
147-
Obj{"s": CommentText([2]string{"#", ""}, "comm")},
113+
Obj{"s": CommentText([2]string{}, "comm")},
148114
CommentNode(false, "comm", nil),
149115
)),
150116

117+
MapSemantic("arg", uast.Argument{}, MapObj(
118+
Obj{
119+
uast.KeyToken: Var("name"),
120+
"default": Var("init"),
121+
},
122+
Obj{
123+
"Name": UASTType(uast.Identifier{}, Obj{
124+
"Name": Var("name"),
125+
}),
126+
"Init": Var("init"),
127+
},
128+
)),
129+
130+
MapSemantic("arg", uast.Argument{}, MapObj(
131+
Obj{
132+
uast.KeyToken: Var("name"),
133+
},
134+
Obj{
135+
"Name": UASTType(uast.Identifier{}, Obj{
136+
"Name": Var("name"),
137+
}),
138+
},
139+
)),
140+
141+
MapSemantic("kwonly_arg", uast.Argument{}, MapObj(
142+
Obj{
143+
uast.KeyToken: Var("name"),
144+
"default": Var("init"),
145+
},
146+
Obj{
147+
"Name": UASTType(uast.Identifier{}, Obj{
148+
"Name": Var("name"),
149+
}),
150+
"Init": Var("init"),
151+
},
152+
)),
153+
154+
MapSemantic("vararg", uast.Argument{}, MapObj(
155+
Obj{
156+
uast.KeyToken: Var("name"),
157+
},
158+
Obj{
159+
"Name": UASTType(uast.Identifier{}, Obj{
160+
"Name": Var("name"),
161+
}),
162+
"Variadic": Bool(true),
163+
},
164+
)),
165+
166+
MapSemantic("kwarg", uast.Argument{}, MapObj(
167+
Obj{
168+
uast.KeyToken: Var("name"),
169+
},
170+
Obj{
171+
"Name": UASTType(uast.Identifier{}, Obj{
172+
"Name": Var("name"),
173+
}),
174+
"MapVariadic": Bool(true),
175+
},
176+
)),
177+
151178
funcDefMap("FunctionDef", false),
152179
funcDefMap("AsyncFunctionDef", true),
153180

181+
// FIXME: check that those are uast:Identifiers too
154182
MapSemantic("Import", uast.RuntimeImport{}, MapObj(
155183
Obj{
156184
"names": Var("names"),

‎driver/normalizer/transforms.go‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,10 @@
1+
package normalizer
2+
3+
import "gopkg.in/bblfsh/sdk.v2/sdk/driver"
4+
5+
var Transforms = driver.Transforms{
6+
Preprocess: Preprocess,
7+
Normalize: Normalize,
8+
Native: Native,
9+
Code: Code,
10+
}

0 commit comments

Comments
 (0)