Skip to content

Commit 0313334

Browse files
juanjuxjuanjux
authored
Merge pull request #35 from juanjux/fix/comments
Remove # from comments' tokens
2 parents df9fabb + 77204f5 commit 0313334

File tree

5 files changed

+338
-245
lines changed

5 files changed

+338
-245
lines changed

‎driver/normalizer/tonode.go‎

Lines changed: 13 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,10 @@
11
package normalizer
22

3-
import "gopkg.in/bblfsh/sdk.v1/uast"
3+
import (
4+
"strings"
5+
6+
"gopkg.in/bblfsh/sdk.v1/uast"
7+
)
48

59
// ToNode is an instance of `uast.ObjectToNode`, defining how to transform an
610
// into a UAST (`uast.Node`).
@@ -13,12 +17,19 @@ var ToNode = &uast.ObjectToNode{
1317
TopLevelIsRootNode: true,
1418
TokenKeys: map[string]bool{"text": true},
1519
Modifier: func(n map[string]interface{}) error {
16-
// Create endOffset as startOffset + textLength
20+
// Create endOffset as startOffset + textLength.
1721
if textEndPos, ok := n["textLength"].(float64); ok {
1822
n["endOffset"] = textEndPos + n["startOffset"].(float64)
1923
delete(n, "textLength")
2024
}
2125

26+
if t, ok := n["elementType"].(string); ok {
27+
if t == "[Bash] Comment" {
28+
if text, ok := n["text"].(string); ok && strings.HasPrefix(text, "#") {
29+
n["text"] = text[1:]
30+
}
31+
}
32+
}
2233
return nil
2334
},
2435
}

‎fixtures/comments.bash‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,5 @@
1+
#!/bin/bash
2+
13
# comment 1
24

35
a="# not a comment"

‎fixtures/comments.bash.native‎

Lines changed: 48 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -4,24 +4,46 @@
44
"errors": [],
55
"ast": {
66
"children": [
7+
{
8+
"children": [
9+
{
10+
"children": [],
11+
"elementType": "[Bash] Shebang",
12+
"startOffset": 0,
13+
"text": "#!/bin/bash\n",
14+
"textLength": 12
15+
}
16+
],
17+
"elementType": "[Bash] shebang element",
18+
"startOffset": 0,
19+
"text": "#!/bin/bash\n",
20+
"textLength": 12
21+
},
22+
{
23+
"children": [],
24+
"elementType": "[Bash] linefeed",
25+
"startOffset": 12,
26+
"text": "\n",
27+
"textLength": 1
28+
},
729
{
830
"children": [],
931
"elementType": "[Bash] Comment",
10-
"startOffset": 0,
32+
"startOffset": 13,
1133
"text": "# comment 1",
1234
"textLength": 11
1335
},
1436
{
1537
"children": [],
1638
"elementType": "[Bash] linefeed",
17-
"startOffset": 11,
39+
"startOffset": 24,
1840
"text": "\n",
1941
"textLength": 1
2042
},
2143
{
2244
"children": [],
2345
"elementType": "[Bash] linefeed",
24-
"startOffset": 12,
46+
"startOffset": 25,
2547
"text": "\n",
2648
"textLength": 1
2749
},
@@ -32,14 +54,14 @@
3254
{
3355
"children": [],
3456
"elementType": "[Bash] assignment_word",
35-
"startOffset": 13,
57+
"startOffset": 26,
3658
"text": "a",
3759
"textLength": 1
3860
},
3961
{
4062
"children": [],
4163
"elementType": "[Bash] =",
42-
"startOffset": 14,
64+
"startOffset": 27,
4365
"text": "=",
4466
"textLength": 1
4567
},
@@ -48,53 +70,53 @@
4870
{
4971
"children": [],
5072
"elementType": "[Bash] string begin",
51-
"startOffset": 15,
73+
"startOffset": 28,
5274
"text": "\"",
5375
"textLength": 1
5476
},
5577
{
5678
"children": [],
5779
"elementType": "[Bash] string content",
58-
"startOffset": 16,
80+
"startOffset": 29,
5981
"text": "# not a comment",
6082
"textLength": 15
6183
},
6284
{
6385
"children": [],
6486
"elementType": "[Bash] string end",
65-
"startOffset": 31,
87+
"startOffset": 44,
6688
"text": "\"",
6789
"textLength": 1
6890
}
6991
],
7092
"elementType": "[Bash] string",
71-
"startOffset": 15,
93+
"startOffset": 28,
7294
"text": "\"# not a comment\"",
7395
"textLength": 17
7496
}
7597
],
7698
"elementType": "var-def-element",
77-
"startOffset": 13,
99+
"startOffset": 26,
78100
"text": "a=\"# not a comment\"",
79101
"textLength": 19
80102
}
81103
],
82104
"elementType": "simple-command",
83-
"startOffset": 13,
105+
"startOffset": 26,
84106
"text": "a=\"# not a comment\"",
85107
"textLength": 19
86108
},
87109
{
88110
"children": [],
89111
"elementType": "[Bash] linefeed",
90-
"startOffset": 32,
112+
"startOffset": 45,
91113
"text": "\n",
92114
"textLength": 1
93115
},
94116
{
95117
"children": [],
96118
"elementType": "[Bash] linefeed",
97-
"startOffset": 33,
119+
"startOffset": 46,
98120
"text": "\n",
99121
"textLength": 1
100122
},
@@ -105,14 +127,14 @@
105127
{
106128
"children": [],
107129
"elementType": "[Bash] assignment_word",
108-
"startOffset": 34,
130+
"startOffset": 47,
109131
"text": "b",
110132
"textLength": 1
111133
},
112134
{
113135
"children": [],
114136
"elementType": "[Bash] =",
115-
"startOffset": 35,
137+
"startOffset": 48,
116138
"text": "=",
117139
"textLength": 1
118140
},
@@ -121,67 +143,67 @@
121143
{
122144
"children": [],
123145
"elementType": "[Bash] string begin",
124-
"startOffset": 36,
146+
"startOffset": 49,
125147
"text": "\"",
126148
"textLength": 1
127149
},
128150
{
129151
"children": [],
130152
"elementType": "[Bash] string content",
131-
"startOffset": 37,
153+
"startOffset": 50,
132154
"text": "foo bar",
133155
"textLength": 7
134156
},
135157
{
136158
"children": [],
137159
"elementType": "[Bash] string end",
138-
"startOffset": 44,
160+
"startOffset": 57,
139161
"text": "\"",
140162
"textLength": 1
141163
}
142164
],
143165
"elementType": "[Bash] string",
144-
"startOffset": 36,
166+
"startOffset": 49,
145167
"text": "\"foo bar\"",
146168
"textLength": 9
147169
}
148170
],
149171
"elementType": "var-def-element",
150-
"startOffset": 34,
172+
"startOffset": 47,
151173
"text": "b=\"foo bar\"",
152174
"textLength": 11
153175
}
154176
],
155177
"elementType": "simple-command",
156-
"startOffset": 34,
178+
"startOffset": 47,
157179
"text": "b=\"foo bar\"",
158180
"textLength": 11
159181
},
160182
{
161183
"children": [],
162184
"elementType": "WHITE_SPACE",
163-
"startOffset": 45,
185+
"startOffset": 58,
164186
"text": " ",
165187
"textLength": 1
166188
},
167189
{
168190
"children": [],
169191
"elementType": "[Bash] Comment",
170-
"startOffset": 46,
192+
"startOffset": 59,
171193
"text": "# comment 2",
172194
"textLength": 11
173195
},
174196
{
175197
"children": [],
176198
"elementType": "[Bash] linefeed",
177-
"startOffset": 57,
199+
"startOffset": 70,
178200
"text": "\n",
179201
"textLength": 1
180202
}
181203
],
182204
"elementType": "FILE",
183205
"startOffset": 0,
184-
"text": "# comment 1\n\na=\"# not a comment\"\n\nb=\"foo bar\" # comment 2\n",
185-
"textLength": 58
206+
"text": "#!/bin/bash\n\n# comment 1\n\na=\"# not a comment\"\n\nb=\"foo bar\" # comment 2\n",
207+
"textLength": 71
186208
}
187209
}

0 commit comments

Comments
 (0)