Skip to content

Commit 8941652

Browse files
pulpdrewpulpdrew
authored
feat: Support separators in floating point literals (#1385)
1 parent 3f97e1a commit 8941652

File tree

6 files changed

+123
-28
lines changed

6 files changed

+123
-28
lines changed

‎NOTICE‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@ under the licensing terms detailed in LICENSE:
2424
* Julien Letellier <[email protected]>
2525
* Guido Zuidhof <[email protected]>
2626
* ncave <[email protected]>
27+
* Andrew Davis <[email protected]>
2728

2829
Portions of this software are derived from third-party works licensed under
2930
the following terms:

‎src/tokenizer.ts‎

Lines changed: 61 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -1494,37 +1494,78 @@ export class Tokenizer extends DiagnosticEmitter {
14941494
}
14951495

14961496
readDecimalFloat(): f64 {
1497-
// TODO: numeric separators (parseFloat can't handle these)
14981497
var text = this.source.text;
1499-
var pos = this.pos;
15001498
var end = this.end;
1501-
var start = pos;
1502-
while (pos < end && isDecimalDigit(text.charCodeAt(pos))) {
1503-
++pos;
1504-
}
1505-
if (pos < end && text.charCodeAt(pos) == CharCode.DOT) {
1506-
++pos;
1507-
while (pos < end && isDecimalDigit(text.charCodeAt(pos))) {
1508-
++pos;
1509-
}
1499+
var start = this.pos;
1500+
var sepCount = 0;
1501+
1502+
sepCount += this.readDecimalFloatPartial(false);
1503+
if (this.pos < end && text.charCodeAt(this.pos) == CharCode.DOT) {
1504+
++this.pos;
1505+
sepCount += this.readDecimalFloatPartial();
15101506
}
1511-
if (pos < end) {
1512-
let c = text.charCodeAt(pos);
1507+
if (this.pos < end) {
1508+
let c = text.charCodeAt(this.pos);
15131509
if ((c | 32) == CharCode.e) {
15141510
if (
1515-
++pos < end &&
1516-
(c = text.charCodeAt(pos)) == CharCode.MINUS || c == CharCode.PLUS &&
1517-
isDecimalDigit(text.charCodeAt(pos + 1))
1511+
++this.pos < end &&
1512+
(c = text.charCodeAt(this.pos)) == CharCode.MINUS || c == CharCode.PLUS &&
1513+
isDecimalDigit(text.charCodeAt(this.pos + 1))
15181514
) {
1519-
++pos;
1515+
++this.pos;
15201516
}
1521-
while (pos < end && isDecimalDigit(text.charCodeAt(pos))) {
1522-
++pos;
1517+
sepCount += this.readDecimalFloatPartial();
1518+
}
1519+
}
1520+
let result = text.substring(start, this.pos);
1521+
if (sepCount > 0) result = result.replaceAll("_", "");
1522+
return parseFloat(result);
1523+
}
1524+
1525+
/** Reads past one section of a decimal float literal. Returns the number of separators encountered. */
1526+
private readDecimalFloatPartial(allowLeadingZeroSep: bool = true): u32 {
1527+
var text = this.source.text;
1528+
var pos = this.pos;
1529+
var start = pos;
1530+
var end = this.end;
1531+
var sepEnd = start;
1532+
var sepCount = 0;
1533+
1534+
while (pos < end) {
1535+
let c = text.charCodeAt(pos);
1536+
1537+
if (c == CharCode._) {
1538+
if (sepEnd == pos) {
1539+
this.error(
1540+
sepEnd == start
1541+
? DiagnosticCode.Numeric_separators_are_not_allowed_here
1542+
: DiagnosticCode.Multiple_consecutive_numeric_separators_are_not_permitted,
1543+
this.range(pos)
1544+
);
1545+
} else if (!allowLeadingZeroSep && pos - 1 == start && text.charCodeAt(pos - 1) == CharCode._0) {
1546+
this.error(
1547+
DiagnosticCode.Numeric_separators_are_not_allowed_here,
1548+
this.range(pos)
1549+
);
15231550
}
1551+
sepEnd = pos + 1;
1552+
++sepCount;
1553+
} else if (!isDecimalDigit(c)) {
1554+
break;
15241555
}
1556+
1557+
++pos;
1558+
}
1559+
1560+
if (pos != start && sepEnd == pos) {
1561+
this.error(
1562+
DiagnosticCode.Numeric_separators_are_not_allowed_here,
1563+
this.range(sepEnd - 1)
1564+
);
15251565
}
1566+
15261567
this.pos = pos;
1527-
return parseFloat(text.substring(start, pos));
1568+
return sepCount;
15281569
}
15291570

15301571
readHexFloat(): f64 {

‎src/tsconfig.json‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,7 @@
44
"outDir": "../out",
55
"allowJs": false,
66
"sourceMap": true,
7-
"target": "ES2016",
7+
"target": "esnext",
88
"strict": true
99
},
1010
"include": [

‎std/portable.json‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,7 @@
77
"downlevelIteration": true,
88
"preserveConstEnums": true,
99
"typeRoots": [ "types" ],
10-
"types": [ "portable" ]
10+
"types": [ "portable" ],
11+
"lib": ["esnext", "esnext.string"]
1112
}
1213
}

‎tests/parser/numeric-separators.ts‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
0b01_01_01;
33
0o12_12_12;
44
0x23_23_23;
5+
1_000_000.1234_1234;
6+
1_0e1_0;
7+
1_000_000e-1_0;
8+
0.0_0;
9+
1_0e0_0;
10+
1_0e0_1;
511

612
// error cases that should still continue parsing:
713

@@ -16,3 +22,19 @@
1622

1723
0x23_23_23_; // 6188
1824
0x23__23_23; // 6189
25+
26+
1000_.1234; // 6188
27+
1000._1234; // 6188
28+
1000.1234_; // 6188
29+
30+
10__00.1234; // 6189
31+
1000.12__34; // 6189
32+
33+
1_e2; // 6188
34+
1e_2; // 6188
35+
1e2_; // 6188
36+
1e-1__0; // 6189
37+
38+
0_0.0; // 6188
39+
0_0.0_0; // 6188
40+
0_0e0_0; // 6188

‎tests/parser/numeric-separators.ts.fixture.ts‎

Lines changed: 36 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -2,6 +2,12 @@
22
21;
33
41610;
44
2302755;
5+
1000000.12341234;
6+
100000000000;
7+
0.0001;
8+
0;
9+
10;
10+
100;
511
111111;
612
111111;
713
21;
@@ -10,11 +16,35 @@
1016
41610;
1117
2302755;
1218
2302755;
13-
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(8,9+0)
14-
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(9,4+0)
15-
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(11,11+0)
16-
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(12,6+0)
17-
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,11+0)
18-
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,6+0)
19+
1000.1234;
20+
1000.1234;
21+
1000.1234;
22+
1000.1234;
23+
1000.1234;
24+
100;
25+
100;
26+
100;
27+
1e-10;
28+
0;
29+
0;
30+
0;
31+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(14,9+0)
32+
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(15,4+0)
1933
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(17,11+0)
2034
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(18,6+0)
35+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(20,11+0)
36+
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(21,6+0)
37+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(23,11+0)
38+
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(24,6+0)
39+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(26,5+0)
40+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(27,6+0)
41+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(28,10+0)
42+
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(30,4+0)
43+
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(31,9+0)
44+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(33,2+0)
45+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(34,3+0)
46+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(35,4+0)
47+
// ERROR 6189: "Multiple consecutive numeric separators are not permitted." in numeric-separators.ts(36,6+0)
48+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(38,2+0)
49+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(39,2+0)
50+
// ERROR 6188: "Numeric separators are not allowed here." in numeric-separators.ts(40,2+0)

0 commit comments

Comments
 (0)