Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit 23bba1d

Browse files
author
Juanjo Alvarez
committed
Create output dir in makefile
Signed-off-by: Juanjo Alvarez <[email protected]>
1 parent 8c6c675 commit 23bba1d

File tree

71 files changed

+1010
-48655
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

71 files changed

+1010
-48655
lines changed

‎Makefile‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@ build-native-internal:
1212
$(PIP3_CMD) ${DEV_DEPS}/python-pydetector/ || $(PIP3_CMD) pydetector-bblfsh
1313
$(PIP2_CMD) ${DEV_DEPS}/python-pydetector/ || $(PIP2_CMD) pydetector-bblfsh
1414
cd native/python_package/ && $(PIP3_CMD) .
15+
mkdir -p $(BUILD_PATH)/bin || true
1516
cp native/sh/native.sh $(BUILD_PATH)/bin/native;
1617
chmod +x $(BUILD_PATH)/bin/native
1718

‎driver/normalizer/annotation.go‎

Lines changed: 3 additions & 57 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,7 @@
11
package normalizer
22

33
import (
4-
"strings"
5-
64
"gopkg.in/bblfsh/sdk.v2/uast"
7-
"gopkg.in/bblfsh/sdk.v2/uast/nodes"
85
"gopkg.in/bblfsh/sdk.v2/uast/role"
96
. "gopkg.in/bblfsh/sdk.v2/uast/transformer"
107
"gopkg.in/bblfsh/sdk.v2/uast/transformer/positioner"
@@ -91,59 +88,6 @@ func loopAnnotate(typ string, mainRole role.Role, roles ...role.Role) Mapping {
9188
}), roles...)
9289
}
9390

94-
func num2dots(n nodes.Value) nodes.Value {
95-
if intval, ok := n.(nodes.Int); ok {
96-
i64val := int(intval)
97-
return nodes.String(strings.Repeat(".", int(i64val)))
98-
}
99-
return n
100-
}
101-
102-
type opLevelDotsNumConv struct {
103-
op Op
104-
orig Op
105-
}
106-
107-
func (op opLevelDotsNumConv) Kinds() nodes.Kind {
108-
return nodes.KindString | nodes.KindInt
109-
}
110-
111-
func (op opLevelDotsNumConv) Check(st *State, n nodes.Node) (bool, error) {
112-
v, ok := n.(nodes.Value)
113-
if !ok {
114-
return false, nil
115-
}
116-
117-
nv := num2dots(v)
118-
res1, err := op.op.Check(st, nv)
119-
if err != nil || !res1 {
120-
return false, err
121-
}
122-
123-
res2, err := op.orig.Check(st, v)
124-
if err != nil || !res2 {
125-
return false, err
126-
}
127-
128-
return res1 && res2, nil
129-
}
130-
131-
func (op opLevelDotsNumConv) Construct(st *State, n nodes.Node) (nodes.Node, error) {
132-
n, err := op.orig.Construct(st, n)
133-
if err != nil {
134-
return nil, err
135-
}
136-
137-
v, ok := n.(nodes.Int)
138-
if !ok {
139-
return nil, ErrExpectedValue.New(n)
140-
}
141-
142-
return v, nil
143-
}
144-
145-
var _ Op = opLevelDotsNumConv{}
146-
14791
var Annotations = []Mapping{
14892
// FIXME: doesnt work
14993
AnnotateType("Module", nil, role.File, role.Module),
@@ -443,7 +387,7 @@ var Annotations = []Mapping{
443387

444388
AnnotateType("ImportFrom", MapObj(Obj{
445389
"module": Var("module"),
446-
"level": opLevelDotsNumConv{op: Var("level"), orig: Var("origlevel")},
390+
"level": OpLevelDotsNumConv{op: Var("level"), orig: Var("origlevel"), prefix: "."},
447391
"names": Var("names"),
448392
}, Obj{
449393
"names": Obj{
@@ -482,7 +426,9 @@ var Annotations = []Mapping{
482426
"decorator_list": Var("decors"),
483427
"body": Var("body_stmts"),
484428
"bases": Var("bases"),
429+
"name": Var("name"),
485430
}, Obj{
431+
uast.KeyToken: Var("name"),
486432
"decorator_list": Obj{
487433
uast.KeyType: String("ClassDef.decorator_list"),
488434
uast.KeyRoles: Roles(role.Type, role.Declaration, role.Call, role.Incomplete),

‎driver/normalizer/normalizer.go‎

Lines changed: 115 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,8 @@ func funcDefMap(typ string, async bool) Mapping {
4343
"async": Bool(async),
4444
},
4545
UASTType(uast.Alias{}, Obj{
46+
// FIXME: can't call identifierWithPos because it would take the position of the
47+
// function node that is not exactly the same as the position of the function name
4648
"Name": UASTType(uast.Identifier{}, Obj{
4749
"Name": Var("name"),
4850
}),
@@ -60,21 +62,34 @@ func funcDefMap(typ string, async bool) Mapping {
6062
))
6163
}
6264

63-
func tokenIsIdentifier(typ, tokenKey string, roles... role.Role) Mapping {
65+
func identifierWithPos(nameVar string) ObjectOp {
66+
return UASTType(uast.Identifier{}, Obj{
67+
uast.KeyPos: UASTType(uast.Positions{}, Obj{
68+
uast.KeyStart: Var(uast.KeyStart),
69+
uast.KeyEnd: Var(uast.KeyEnd),
70+
}),
71+
"Name": Var(nameVar),
72+
})
73+
}
74+
75+
func tokenIsIdentifier(typ, tokenKey string, roles ...role.Role) Mapping {
6476
return AnnotateType(typ, MapObj(
6577
Fields{
6678
{Name: tokenKey, Op: Var("name")},
6779
},
6880
Fields{
69-
{Name: tokenKey, Op: UASTType(uast.Identifier{}, Obj{
70-
"Name": Var("name"),
71-
})},
81+
{Name: tokenKey,
82+
Op: UASTType(uast.Identifier{}, Obj{
83+
"Name": Var("name"),
84+
})},
7285
}),
7386
roles...)
7487
}
7588

89+
7690
var Normalizers = []Mapping{
7791

92+
// FIXME: no positions for keywords in the native AST
7893
tokenIsIdentifier("keyword", "arg", role.Name),
7994

8095
MapSemantic("Str", uast.String{}, MapObj(
@@ -117,21 +132,6 @@ var Normalizers = []Mapping{
117132
Obj{"Name": Var("name")},
118133
)),
119134

120-
MapSemantic("alias", uast.Alias{}, MapObj(
121-
Obj{
122-
"name": Var("name"),
123-
"asname": Var("aliased"),
124-
},
125-
Obj{
126-
"Name": UASTType(uast.Identifier{}, Obj{
127-
"Name": Var("name"),
128-
}),
129-
"Node": UASTType(uast.Identifier{}, Obj{
130-
"Name": Var("aliased"),
131-
}),
132-
},
133-
)),
134-
135135
MapSemantic("Name", uast.Identifier{}, MapObj(
136136
Obj{"attr": Var("name")},
137137
Obj{"Name": Var("name")},
@@ -153,9 +153,7 @@ var Normalizers = []Mapping{
153153
"default": Var("init"),
154154
},
155155
Obj{
156-
"Name": UASTType(uast.Identifier{}, Obj{
157-
"Name": Var("name"),
158-
}),
156+
"Name": identifierWithPos("name"),
159157
"Init": Var("init"),
160158
},
161159
)),
@@ -165,9 +163,7 @@ var Normalizers = []Mapping{
165163
uast.KeyToken: Var("name"),
166164
},
167165
Obj{
168-
"Name": UASTType(uast.Identifier{}, Obj{
169-
"Name": Var("name"),
170-
}),
166+
"Name": identifierWithPos("name"),
171167
},
172168
)),
173169

@@ -177,10 +173,8 @@ var Normalizers = []Mapping{
177173
"default": Var("init"),
178174
},
179175
Obj{
180-
"Name": UASTType(uast.Identifier{}, Obj{
181-
"Name": Var("name"),
182-
}),
183176
"Init": Var("init"),
177+
"Name": identifierWithPos("name"),
184178
},
185179
)),
186180

@@ -189,9 +183,7 @@ var Normalizers = []Mapping{
189183
uast.KeyToken: Var("name"),
190184
},
191185
Obj{
192-
"Name": UASTType(uast.Identifier{}, Obj{
193-
"Name": Var("name"),
194-
}),
186+
"Name": identifierWithPos("name"),
195187
"Variadic": Bool(true),
196188
},
197189
)),
@@ -201,16 +193,43 @@ var Normalizers = []Mapping{
201193
uast.KeyToken: Var("name"),
202194
},
203195
Obj{
204-
"Name": UASTType(uast.Identifier{}, Obj{
205-
"Name": Var("name"),
206-
}),
196+
"Name": identifierWithPos("name"),
207197
"MapVariadic": Bool(true),
208198
},
209199
)),
210200

211201
funcDefMap("FunctionDef", false),
212202
funcDefMap("AsyncFunctionDef", true),
213203

204+
AnnotateType("ClassDef", MapObj(Obj{
205+
"decorator_list": Var("decors"),
206+
"body": Var("body_stmts"),
207+
"bases": Var("bases"),
208+
"name": Var("name"),
209+
uast.KeyPos: Obj{
210+
uast.KeyType: String(uast.KeyPos),
211+
uast.KeyStart: Var(uast.KeyStart),
212+
uast.KeyEnd: Var(uast.KeyEnd),
213+
},
214+
}, Obj{
215+
uast.KeyToken: identifierWithPos("name"),
216+
"decorator_list": Obj{
217+
uast.KeyType: String("ClassDef.decorator_list"),
218+
uast.KeyRoles: Roles(role.Type, role.Declaration, role.Call, role.Incomplete),
219+
"decorators": Var("decors"),
220+
},
221+
"body": Obj{
222+
uast.KeyType: String("ClassDef.body"),
223+
uast.KeyRoles: Roles(role.Type, role.Declaration, role.Body),
224+
"body_stmts": Var("body_stmts"),
225+
},
226+
"bases": Obj{
227+
uast.KeyType: String("ClassDef.bases"),
228+
uast.KeyRoles: Roles(role.Type, role.Declaration, role.Base),
229+
"bases": Var("bases"),
230+
},
231+
}), role.Type, role.Declaration, role.Identifier, role.Statement),
232+
214233
AnnotateType("Import", MapObj(
215234
Obj{
216235
"names": Each("vals", Var("name")),
@@ -223,29 +242,76 @@ var Normalizers = []Mapping{
223242
},
224243
), role.Import, role.Declaration, role.Statement),
225244

226-
// FIXME: what to do with levels? convert to ../../... in Path?
227-
// FIXME: "import * from x": check the * and set "All" to true
228-
//MapSemantic("ImportFrom", uast.RuntimeImport{}, MapObj(
229-
// Obj{
230-
// "names": Check(Is(Arr(Any))),
231-
// "module": Var("module"),
232-
// },
233-
// Obj{
234-
// "All": Bool(true),
235-
// "Path": UASTType(uast.Identifier{}, Obj{
236-
// "Name": Var("module"),
237-
// }),
238-
// },
239-
//)),
245+
// FIXME: aliases doesn't have a position (can't be currently fixed by the tokenizer
246+
// because they don't even have a line in the native AST)
247+
MapSemantic("alias", uast.Alias{}, MapObj(
248+
Obj{
249+
"name": Var("name"),
250+
"asname": Cases("case_alias",
251+
Check(Is(nil), Var("nilalias")),
252+
Check(Not(Is(nil)), Var("alias")),
253+
),
254+
},
255+
CasesObj("case_alias",
256+
Obj{
257+
"Name": UASTType(uast.Identifier{}, Obj{
258+
"Name": Var("name"),
259+
}),
260+
},
261+
Objs{
262+
{"Node": Obj{}},
263+
{
264+
//"Node": identifierWithPos("name"),
265+
"Node": UASTType(uast.Identifier{}, Obj{ "Name": Var("alias"), }),
266+
}},
267+
))),
268+
269+
// Star imports
270+
MapSemantic("ImportFrom", uast.RuntimeImport{}, MapObj(
271+
Obj{
272+
"names": Arr(
273+
Obj{
274+
uast.KeyType: String("uast:Alias"),
275+
uast.KeyPos: Var("pos"),
276+
"Name": Obj{
277+
uast.KeyType: String("uast:Identifier"),
278+
"Name": String("*"),
279+
},
280+
"Node": Obj{},
281+
},
282+
),
283+
"level": Var("level"),
284+
"module": Var("module"),
285+
},
286+
Obj{
287+
"All": Bool(true),
288+
"Path": UASTType(uast.Identifier{}, Obj{
289+
"Name": OpPrependPath{
290+
// FIXME: no position for the module (path) in the native AST, only when the import starts
291+
numLevel: Var("level"),
292+
path: Var("module"),
293+
joined: Var("joined"),
294+
prefix: "../",
295+
},
296+
}),
297+
},
298+
)),
299+
240300
MapSemantic("ImportFrom", uast.RuntimeImport{}, MapObj(
241301
Obj{
242302
"names": Var("names"),
243303
"module": Var("module"),
304+
"level": Var("level"),
244305
},
245306
Obj{
246307
"Names": Var("names"),
247308
"Path": UASTType(uast.Identifier{}, Obj{
248-
"Name": Var("module"),
309+
"Name": OpPrependPath{
310+
numLevel: Var("level"),
311+
path: Var("module"),
312+
joined: Var("joined"),
313+
prefix: "../",
314+
},
249315
}),
250316
},
251317
)),

0 commit comments

Comments
 (0)