Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit 58b2fc6

Browse files
author
Juanjo Alvarez
committed
Fix some corner cases for comments
Signed-off-by: Juanjo Alvarez <[email protected]>
1 parent 8350cc8 commit 58b2fc6

File tree

117 files changed

+62849
-41474
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

117 files changed

+62849
-41474
lines changed

‎Gopkg.lock‎

Lines changed: 17 additions & 105 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎Gopkg.toml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,7 +3,7 @@
33

44
[[constraint]]
55
name = "gopkg.in/bblfsh/sdk.v2"
6-
version = "2.12.x"
6+
version = "2.13.x"
77

88
[prune]
99
go-tests = true

‎driver/fixtures/fixtures_test.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ var Suite = &fixtures.Suite{
2727
Semantic: fixtures.SemanticConfig{
2828
BlacklistTypes: []string{
2929
"AsyncFunctionDef",
30+
"BoolLiteral",
3031
"Bytes",
3132
"FunctionDef",
3233
"ImportFrom",
@@ -35,11 +36,11 @@ var Suite = &fixtures.Suite{
3536
"NoopSameLine",
3637
"Str",
3738
"StringLiteral",
39+
"alias",
3840
"arg",
3941
"kwarg",
4042
"kwonly_arg",
4143
"vararg",
42-
"alias",
4344
},
4445
},
4546
}

‎driver/normalizer/annotation.go‎

Lines changed: 26 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -5,6 +5,7 @@ import (
55
"gopkg.in/bblfsh/sdk.v2/uast/role"
66
. "gopkg.in/bblfsh/sdk.v2/uast/transformer"
77
"gopkg.in/bblfsh/sdk.v2/uast/transformer/positioner"
8+
"strings"
89
)
910

1011
var Native = Transformers([][]Transformer{
@@ -88,6 +89,21 @@ func loopAnnotate(typ string, mainRole role.Role, roles ...role.Role) Mapping {
8889
}), roles...)
8990
}
9091

92+
func uncomment_bash(s string) (string, error) {
93+
if strings.HasPrefix(s, "#") {
94+
s = s[1:]
95+
}
96+
return s, nil
97+
}
98+
99+
func comment_bash(s string) (string, error) {
100+
return "#" + s, nil
101+
}
102+
103+
func UncommentBashLike(vr string) Op {
104+
return StringConv(Var(vr), uncomment_bash, comment_bash)
105+
}
106+
91107
var Annotations = []Mapping{
92108
// FIXME: doesnt work
93109
AnnotateType("Module", nil, role.File, role.Module),
@@ -356,20 +372,17 @@ var Annotations = []Mapping{
356372
"lines": {Arr: true, Roles: role.Roles{role.Noop}},
357373
}, role.Noop),
358374

359-
AnnotateType("NoopLine", FieldRoles{
360-
"noop_line": {Rename: uast.KeyToken},
361-
}, role.Noop, role.Comment),
362-
363-
AnnotateType("NoopSameLine", FieldRoles{
364-
"s": {Rename: uast.KeyToken},
365-
}, role.Noop, role.Comment),
375+
AnnotateType("NoopLine", MapObj(Obj{
376+
"noop_line": UncommentBashLike("txt"),
377+
}, Obj{
378+
uast.KeyToken: Var("txt"),
379+
}), role.Comment, role.Noop),
366380

367-
// Qualified Identifiers
368-
// FIXME XXX remove
369-
// a.b.c ("a" and "b" will be Qualified+Identifier, "c" will be just Identifier)
370-
//AnnotateType("Attribute", FieldRoles{
371-
// "value": {Opt: true, Roles: role.Roles{role.Qualified}},
372-
//}),
381+
AnnotateType("NoopSameLine", MapObj(Obj{
382+
"s": UncommentBashLike("txt"),
383+
}, Obj{
384+
uast.KeyToken: Var("txt"),
385+
}), role.Comment, role.Noop),
373386

374387
// Import
375388
AnnotateType("Import", nil, role.Import, role.Declaration, role.Statement),

‎driver/normalizer/normalizer.go‎

Lines changed: 88 additions & 87 deletions
Original file line numberDiff line numberDiff line change
@@ -72,109 +72,110 @@ func identifierWithPos(nameVar string) ObjectOp {
7272
})
7373
}
7474

75-
func tokenIsIdentifier(typ, tokenKey string, roles ...role.Role) Mapping {
76-
return AnnotateType(typ, MapObj(
77-
Fields{
78-
{Name: tokenKey, Op: Var("name")},
79-
},
80-
Fields{
81-
{Name: tokenKey,
82-
Op: UASTType(uast.Identifier{}, Obj{
83-
"Name": Var("name"),
84-
})},
75+
func mapStr(nativeType string) Mapping {
76+
return Map(
77+
Part("_", Fields{
78+
{Name: uast.KeyType, Op: String(nativeType)},
79+
{Name: uast.KeyPos, Op: Var("pos_")},
80+
{Name: "s", Op: Var("s")},
81+
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
82+
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
83+
}),
84+
Part("_", Fields{
85+
{Name: uast.KeyType, Op: String("Boxed" + nativeType)},
86+
{Name: "boxed_value", Op: UASTType(uast.String{}, Obj{
87+
uast.KeyPos: Var("pos_"),
88+
"Value": Var("s"),
89+
"Format": String(""),
90+
})},
91+
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
92+
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
8593
}),
86-
roles...)
94+
)
8795
}
8896

89-
9097
var Normalizers = []Mapping{
9198

92-
// FIXME: no positions for keywords in the native AST
93-
tokenIsIdentifier("keyword", "arg", role.Name),
99+
// Box Names, Strings, and Bools into a "BoxedFoo" moving the real node to the
100+
// "value" property and keeping the comments in the parent (if not, comments would be lost
101+
// when promoting the objects)
102+
Map(
103+
Part("_", Fields{
104+
{Name: uast.KeyType, Op: String("Name")},
105+
{Name: uast.KeyPos, Op: Var("pos_")},
106+
{Name: "id", Op: Var("id")},
107+
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
108+
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
109+
}),
110+
Part("_", Fields{
111+
{Name: uast.KeyType, Op: String("BoxedName")},
112+
{Name: "boxed_value", Op: UASTType(uast.Identifier{}, Obj{
113+
uast.KeyPos: Var("pos_"),
114+
"Name": Var("id"),
115+
})},
116+
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
117+
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
118+
}),
119+
),
94120

95-
MapSemantic("Str", uast.String{}, MapObj(
96-
Obj{
97-
"s": Var("val"),
98-
},
99-
Obj{
100-
"Value": Var("val"),
101-
"Format": String(""),
102-
},
103-
)),
121+
// TODO: uncomment after SDK 2.13.x update
122+
// (upgrade currently blocked by: https://github.com/bblfsh/sdk/issues/353)
123+
Map(
124+
Part("_", Fields{
125+
{Name: uast.KeyType, Op: String("BoolLiteral")},
126+
{Name: uast.KeyPos, Op: Var("pos_")},
127+
//{Name: "LiteralValue", Op: Var("lv")},
128+
{Name: "value", Op: Var("lv")},
129+
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
130+
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
131+
}),
132+
Part("_", Fields{
133+
{Name: uast.KeyType, Op: String("BoxedBoolLiteral")},
134+
{Name: "boxed_value", Op: UASTType(uast.Bool{}, Obj{
135+
uast.KeyPos: Var("pos_"),
136+
"Value": Var("lv"),
137+
})},
138+
{Name: "noops_previous", Optional: "np_opt", Op: Var("noops_previous")},
139+
{Name: "noops_sameline", Optional: "ns_opt", Op: Var("noops_sameline")},
140+
}),
141+
),
104142

105-
MapSemantic("Bytes", uast.String{}, MapObj(
106-
Obj{
107-
"s": Var("val"),
108-
},
143+
mapStr("Bytes"),
144+
mapStr("Str"),
145+
mapStr("StringLiteral"),
146+
147+
MapSemantic("NoopLine", uast.Comment{}, MapObj(
109148
Obj{
110-
"Value": Var("val"),
111-
"Format": String(""),
149+
"noop_line": CommentText([2]string{"#", ""}, "comm"),
112150
},
151+
CommentNode(false, "comm", nil),
113152
)),
114153

115-
MapSemantic("StringLiteral", uast.String{}, MapObj(
116-
Obj{
117-
"s": Var("val"),
118-
},
154+
MapSemantic("NoopSameLine", uast.Comment{}, MapObj(
119155
Obj{
120-
"Value": Var("val"),
121-
"Format": String(""),
156+
"s": CommentText([2]string{"#", ""}, "comm"),
122157
},
158+
CommentNode(false, "comm", nil),
123159
)),
124160

125-
MapSemantic("Name", uast.Identifier{}, MapObj(
126-
Obj{"id": Var("name")},
127-
Obj{"Name": Var("name")},
128-
)),
161+
// FIXME: no positions for keywords in the native AST
162+
AnnotateType("keyword", MapObj(
163+
Fields{
164+
{Name: "arg", Op: Var("name")},
165+
},
166+
Fields{
167+
{Name: "arg",
168+
Op: UASTType(uast.Identifier{}, Obj{
169+
"Name": Var("name"),
170+
})},
171+
}),
172+
role.Name),
129173

130174
MapSemantic("Attribute", uast.Identifier{}, MapObj(
131175
Obj{"attr": Var("name")},
132176
Obj{"Name": Var("name")},
133177
)),
134178

135-
MapSemantic("Name", uast.Identifier{}, MapObj(
136-
Obj{"attr": Var("name")},
137-
Obj{"Name": Var("name")},
138-
)),
139-
140-
MapSemantic("NoopLine", uast.Comment{}, MapObj(
141-
Obj{"noop_line": CommentText([2]string{}, "comm")},
142-
CommentNode(false, "comm", nil),
143-
)),
144-
145-
//SameLineNoops like the other comment container nodes hold an array of lines, but by
146-
//definition it can only hold one, thus we copy the position from the parent to the (only) child
147-
//that doesn't have it
148-
MapSemantic("SameLineNoops", uast.Comment{}, MapObj(
149-
Obj{
150-
"noop_lines": Arr(
151-
Obj{
152-
uast.KeyType: String("NoopSameLine"),
153-
uast.KeyPos: Var("foo"),
154-
"s": CommentText([2]string{}, "comm"),
155-
},
156-
),
157-
},
158-
CommentNode(false, "comm", nil),
159-
)),
160-
MapSemantic("SameLineNoops", uast.Comment{}, MapObj(
161-
Obj{
162-
"noop_lines": Arr(
163-
Obj{
164-
uast.KeyType: String("NoopSameLine"),
165-
"s": CommentText([2]string{}, "comm"),
166-
},
167-
),
168-
},
169-
CommentNode(false, "comm", nil),
170-
)),
171-
172-
// XXX remove
173-
//MapSemantic("NoopSameLine", uast.Comment{}, MapObj(
174-
// Obj{"s": CommentText([2]string{}, "comm")},
175-
// CommentNode(false, "comm", nil),
176-
//)),
177-
178179
MapSemantic("arg", uast.Argument{}, MapObj(
179180
Obj{
180181
uast.KeyToken: Var("name"),
@@ -211,7 +212,7 @@ var Normalizers = []Mapping{
211212
uast.KeyToken: Var("name"),
212213
},
213214
Obj{
214-
"Name": identifierWithPos("name"),
215+
"Name": identifierWithPos("name"),
215216
"Variadic": Bool(true),
216217
},
217218
)),
@@ -221,7 +222,7 @@ var Normalizers = []Mapping{
221222
uast.KeyToken: Var("name"),
222223
},
223224
Obj{
224-
"Name": identifierWithPos("name"),
225+
"Name": identifierWithPos("name"),
225226
"MapVariadic": Bool(true),
226227
},
227228
)),
@@ -260,7 +261,7 @@ var Normalizers = []Mapping{
260261
Objs{
261262
{"Node": Obj{}},
262263
{
263-
"Node": UASTType(uast.Identifier{}, Obj{ "Name": Var("alias"), }),
264+
"Node": UASTType(uast.Identifier{}, Obj{"Name": Var("alias")}),
264265
}},
265266
))),
266267

@@ -270,10 +271,10 @@ var Normalizers = []Mapping{
270271
"names": Arr(
271272
Obj{
272273
uast.KeyType: String("uast:Alias"),
273-
uast.KeyPos: Var("pos"),
274+
uast.KeyPos: Var("pos"),
274275
"Name": Obj{
275276
uast.KeyType: String("uast:Identifier"),
276-
"Name": String("*"),
277+
"Name": String("*"),
277278
},
278279
"Node": Obj{},
279280
},

0 commit comments

Comments
 (0)