Skip to content
This repository was archived by the owner on Mar 8, 2020. It is now read-only.

Commit e696d7e

Browse files
Denys Smirnovdennwc
Denys Smirnov
authored and
dennwc
committed
interpret offset as a utf-16 code units; fixes #71
Signed-off-by: Denys Smirnov <[email protected]>
1 parent e8ce7b8 commit e696d7e

File tree

6 files changed

+207
-4
lines changed

6 files changed

+207
-4
lines changed

‎Gopkg.lock‎

Lines changed: 3 additions & 3 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎driver/normalizer/annotation.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -23,7 +23,7 @@ var Native = Transformers([][]Transformer{
2323
// PreprocessCode is a preprocessor stage that can use the source code to
2424
// fix tokens and positional information.
2525
var PreprocessCode = []CodeTransformer{
26-
positioner.FromUnicodeOffset(),
26+
positioner.FromUTF16Offset(),
2727
}
2828

2929
var Code []CodeTransformer // TODO(dennwc): deprecated

‎fixtures/unicode.js‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
"𝓏"

‎fixtures/unicode.js.native‎

Lines changed: 70 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,70 @@
1+
{
2+
comments: [],
3+
end: 4,
4+
loc: {
5+
end: {
6+
column: 4,
7+
line: 1,
8+
},
9+
start: {
10+
column: 0,
11+
line: 1,
12+
},
13+
},
14+
program: {
15+
body: [],
16+
directives: [
17+
{
18+
end: 4,
19+
loc: {
20+
end: {
21+
column: 4,
22+
line: 1,
23+
},
24+
start: {
25+
column: 0,
26+
line: 1,
27+
},
28+
},
29+
start: 0,
30+
type: "Directive",
31+
value: {
32+
end: 4,
33+
extra: {
34+
raw: "\"𝓏\"",
35+
rawValue: "𝓏",
36+
},
37+
loc: {
38+
end: {
39+
column: 4,
40+
line: 1,
41+
},
42+
start: {
43+
column: 0,
44+
line: 1,
45+
},
46+
},
47+
start: 0,
48+
type: "DirectiveLiteral",
49+
value: "𝓏",
50+
},
51+
},
52+
],
53+
end: 4,
54+
loc: {
55+
end: {
56+
column: 4,
57+
line: 1,
58+
},
59+
start: {
60+
column: 0,
61+
line: 1,
62+
},
63+
},
64+
sourceType: "module",
65+
start: 0,
66+
type: "Program",
67+
},
68+
start: 0,
69+
type: "File",
70+
}

‎fixtures/unicode.js.sem.uast‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{ '@type': "javascript:File",
2+
'@role': [File],
3+
'@pos': { '@type': "uast:Positions",
4+
start: { '@type': "uast:Position",
5+
offset: 0,
6+
line: 1,
7+
col: 1,
8+
},
9+
end: { '@type': "uast:Position",
10+
offset: 6,
11+
line: 1,
12+
col: 7,
13+
},
14+
},
15+
comments: [],
16+
program: { '@type': "javascript:Program",
17+
'@role': [Module],
18+
'@pos': { '@type': "uast:Positions",
19+
start: { '@type': "uast:Position",
20+
offset: 0,
21+
line: 1,
22+
col: 1,
23+
},
24+
end: { '@type': "uast:Position",
25+
offset: 6,
26+
line: 1,
27+
col: 7,
28+
},
29+
},
30+
body: [],
31+
directives: [
32+
{ '@type': "javascript:Directive",
33+
'@role': [Incomplete],
34+
'@pos': { '@type': "uast:Positions",
35+
start: { '@type': "uast:Position",
36+
offset: 0,
37+
line: 1,
38+
col: 1,
39+
},
40+
end: { '@type': "uast:Position",
41+
offset: 6,
42+
line: 1,
43+
col: 7,
44+
},
45+
},
46+
value: { '@type': "javascript:DirectiveLiteral",
47+
'@token': "𝓏",
48+
'@role': [Expression, Incomplete, Literal],
49+
'@pos': { '@type': "uast:Positions",
50+
start: { '@type': "uast:Position",
51+
offset: 0,
52+
line: 1,
53+
col: 1,
54+
},
55+
end: { '@type': "uast:Position",
56+
offset: 6,
57+
line: 1,
58+
col: 7,
59+
},
60+
},
61+
},
62+
},
63+
],
64+
sourceType: "module",
65+
},
66+
}

‎fixtures/unicode.js.uast‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,66 @@
1+
{ '@type': "File",
2+
'@role': [File],
3+
'@pos': { '@type': "uast:Positions",
4+
start: { '@type': "uast:Position",
5+
offset: 0,
6+
line: 1,
7+
col: 1,
8+
},
9+
end: { '@type': "uast:Position",
10+
offset: 6,
11+
line: 1,
12+
col: 7,
13+
},
14+
},
15+
comments: [],
16+
program: { '@type': "Program",
17+
'@role': [Module],
18+
'@pos': { '@type': "uast:Positions",
19+
start: { '@type': "uast:Position",
20+
offset: 0,
21+
line: 1,
22+
col: 1,
23+
},
24+
end: { '@type': "uast:Position",
25+
offset: 6,
26+
line: 1,
27+
col: 7,
28+
},
29+
},
30+
body: [],
31+
directives: [
32+
{ '@type': "Directive",
33+
'@role': [Incomplete],
34+
'@pos': { '@type': "uast:Positions",
35+
start: { '@type': "uast:Position",
36+
offset: 0,
37+
line: 1,
38+
col: 1,
39+
},
40+
end: { '@type': "uast:Position",
41+
offset: 6,
42+
line: 1,
43+
col: 7,
44+
},
45+
},
46+
value: { '@type': "DirectiveLiteral",
47+
'@token': "𝓏",
48+
'@role': [Expression, Incomplete, Literal],
49+
'@pos': { '@type': "uast:Positions",
50+
start: { '@type': "uast:Position",
51+
offset: 0,
52+
line: 1,
53+
col: 1,
54+
},
55+
end: { '@type': "uast:Position",
56+
offset: 6,
57+
line: 1,
58+
col: 7,
59+
},
60+
},
61+
},
62+
},
63+
],
64+
sourceType: "module",
65+
},
66+
}

0 commit comments

Comments
 (0)