Skip to content

Commit 5156c3d

Browse files
jogibear9988sebastienros
jogibear9988
and
sebastienros
authored
BigInt & numeric separator support (#204)
* bigint & numeric seperator support * perfomance optimations and a test * fix brakets * Update Literal.cs Co-authored-by: Sébastien Ros <[email protected]>
1 parent dabc61f commit 5156c3d

File tree

9 files changed

+365
-281
lines changed

9 files changed

+365
-281
lines changed

‎src/Esprima/Ast/BigIntLiteral.cs‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
using System.Numerics;
2+
3+
namespace Esprima.Ast
4+
{
5+
public sealed class BigIntLiteral : Literal
6+
{
7+
public readonly string BigInt;
8+
9+
public BigInteger? BigIntValue => (BigInteger?) Value;
10+
11+
public BigIntLiteral(BigInteger value, string raw) : base(TokenType.BigIntLiteral, value, raw)
12+
{
13+
BigInt = raw;
14+
}
15+
}
16+
}

‎src/Esprima/Ast/Literal.cs‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,16 @@
1-
using System.Text.RegularExpressions;
1+
using System.Numerics;
2+
using System.Text.RegularExpressions;
23
using Esprima.Utils;
34

45
namespace Esprima.Ast
56
{
6-
public sealed class Literal : Expression
7+
public class Literal : Expression
78
{
89
public string? StringValue => TokenType == TokenType.StringLiteral ? Value as string : null;
910
public readonly double NumericValue;
1011
public bool BooleanValue => TokenType == TokenType.BooleanLiteral && NumericValue != 0;
1112
public Regex? RegexValue => TokenType == TokenType.RegularExpression ? (Regex?) Value : null;
13+
public BigInteger? BigIntValue => TokenType == TokenType.BigIntLiteral ? (BigInteger?) Value : null;
1214

1315
public readonly RegexValue? Regex;
1416
public readonly object? Value;

‎src/Esprima/Character.cs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -100,7 +100,7 @@ public static bool IsHexDigit(char cp)
100100
return cp >= '0' && cp <= '9' ||
101101
cp >= 'A' && cp <= 'F' ||
102102
cp >= 'a' && cp <= 'f';
103-
}
103+
}
104104

105105
public static bool IsOctalDigit(char cp)
106106
{

‎src/Esprima/JavascriptParser.cs‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -590,6 +590,19 @@ private Expression ParsePrimaryExpression()
590590
token = NextToken();
591591
raw = GetTokenRaw(token);
592592
expr = Finalize(node, new Literal(token.NumericValue, raw));
593+
break;
594+
595+
case TokenType.BigIntLiteral:
596+
if (_context.Strict && _lookahead.Octal)
597+
{
598+
TolerateUnexpectedToken(_lookahead, Messages.StrictOctalLiteral);
599+
}
600+
601+
_context.IsAssignmentTarget = false;
602+
_context.IsBindingElement = false;
603+
token = NextToken();
604+
raw = GetTokenRaw(token);
605+
expr = Finalize(node, new BigIntLiteral(token.BigIntValue.Value, raw));
593606
break;
594607

595608
case TokenType.BooleanLiteral:
@@ -864,6 +877,16 @@ private Expression ParseObjectPropertyKey(Boolean isPrivate = false)
864877

865878
raw = GetTokenRaw(token);
866879
key = Finalize(node, new Literal(token.NumericValue, raw));
880+
break;
881+
882+
case TokenType.BigIntLiteral:
883+
if (_context.Strict && token.Octal)
884+
{
885+
TolerateUnexpectedToken(token, Messages.StrictOctalLiteral);
886+
}
887+
888+
raw = GetTokenRaw(token);
889+
key = Finalize(node, new BigIntLiteral(token.BigIntValue.Value, raw));
867890
break;
868891

869892
case TokenType.Identifier:

‎src/Esprima/Messages.cs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@ public static class Messages
4040
public const string MultipleDefaultsInSwitch = "More than one default clause in switch statement";
4141
public const string NewlineAfterThrow = "Illegal newline after throw";
4242
public const string NoAsAfterImportNamespace = "Unexpected token";
43-
public const string NoCatchOrFinally = "Missing catch or finally after try";
43+
public const string NoCatchOrFinally = "Missing catch or finally after try";
44+
public const string NumericSeperatorOneUnderscore = "Numeric separator must be exactly one underscore";
45+
public const string NumericSeperatorNotAllowedHere = "Numeric separator is not allowed here";
4446
public const string ParameterAfterRestParameter = "Rest parameter must be last formal parameter";
4547
public const string PropertyAfterRestProperty = "Unexpected token";
4648
public const string Redeclaration = "{0} \"{1}\" has already been declared";

‎src/Esprima/Scanner.cs‎

Lines changed: 66 additions & 56 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
using System;
22
using System.Collections.Generic;
33
using System.Globalization;
4+
using System.Numerics;
45
using System.Runtime.CompilerServices;
56
using System.Text;
67
using System.Text.RegularExpressions;
@@ -887,20 +888,10 @@ static string SafeSubstring(string s, int startIndex, int length)
887888
// https://tc39.github.io/ecma262/#sec-literals-numeric-literals
888889

889890
public Token ScanHexLiteral(int start)
890-
{
891-
var index = Index;
892-
893-
while (!Eof())
894-
{
895-
if (!Character.IsHexDigit(Source.CharCodeAt(Index)))
896-
{
897-
break;
898-
}
899-
900-
Index++;
901-
}
902-
903-
var number = Source.Substring(index, Index - index);
891+
{
892+
var sb = GetStringBuilder();
893+
this.ScanLiteralPart(sb, Character.IsHexDigit);
894+
var number = sb.ToString();
904895

905896
if (number.Length == 0)
906897
{
@@ -957,22 +948,11 @@ public Token ScanHexLiteral(int start)
957948
}
958949

959950
public Token ScanBinaryLiteral(int start)
960-
{
951+
{
961952
char ch;
962-
var index = Index;
963-
964-
while (!Eof())
965-
{
966-
ch = Source[Index];
967-
if (ch != '0' && ch != '1')
968-
{
969-
break;
970-
}
971-
972-
Index++;
973-
}
974-
975-
var number = Source.Substring(index, Index - index);
953+
var sb = GetStringBuilder();
954+
this.ScanLiteralPart(sb, c => c == '0' || c == '1');
955+
var number = sb.ToString();
976956

977957
if (number.Length == 0)
978958
{
@@ -1015,18 +995,9 @@ public Token ScanOctalLiteral(char prefix, int start)
1015995
else
1016996
{
1017997
++Index;
1018-
}
1019-
1020-
while (!Eof())
1021-
{
1022-
if (!Character.IsOctalDigit(Source.CharCodeAt(Index)))
1023-
{
1024-
break;
1025-
}
1026-
1027-
sb.Append(Source[Index++]);
1028-
}
1029-
998+
}
999+
1000+
this.ScanLiteralPart(sb, Character.IsOctalDigit);
10301001
var number = sb.ToString();
10311002

10321003
if (!octal && number.Length == 0)
@@ -1084,6 +1055,40 @@ public bool IsImplicitOctalLiteral()
10841055
return true;
10851056
}
10861057

1058+
private void ScanLiteralPart(StringBuilder sb, Func<char, bool> check)
1059+
{
1060+
var charCode = Source.CharCodeAt(Index);
1061+
if (charCode == '_')
1062+
{
1063+
ThrowUnexpectedToken(Messages.NumericSeperatorNotAllowedHere);
1064+
}
1065+
1066+
while ((check(charCode) || charCode == '_'))
1067+
{
1068+
if (charCode != '_')
1069+
{
1070+
sb.Append(charCode);
1071+
}
1072+
Index++;
1073+
var newCharCode = Source.CharCodeAt(Index);
1074+
if (charCode == '_' && newCharCode == '_')
1075+
{
1076+
ThrowUnexpectedToken(Messages.NumericSeperatorOneUnderscore);
1077+
}
1078+
1079+
if (Eof())
1080+
{
1081+
break;
1082+
}
1083+
charCode = newCharCode;
1084+
}
1085+
1086+
if (charCode == '_')
1087+
{
1088+
ThrowUnexpectedToken(Messages.NumericSeperatorNotAllowedHere);
1089+
}
1090+
}
1091+
10871092
public Token ScanNumericLiteral()
10881093
{
10891094
var sb = GetStringBuilder();
@@ -1095,7 +1100,6 @@ public Token ScanNumericLiteral()
10951100
if (ch != '.')
10961101
{
10971102
var first = Source[Index++];
1098-
sb.Append(first);
10991103
ch = Source.CharCodeAt(Index);
11001104

11011105
// Hex number starts with '0x'.
@@ -1130,21 +1134,15 @@ public Token ScanNumericLiteral()
11301134
}
11311135
}
11321136

1133-
while (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
1134-
{
1135-
sb.Append(Source[Index++]);
1136-
}
1137-
1137+
--Index;
1138+
this.ScanLiteralPart(sb, Character.IsDecimalDigit);
11381139
ch = Source.CharCodeAt(Index);
11391140
}
11401141

11411142
if (ch == '.')
11421143
{
1143-
sb.Append(Source[Index++]);
1144-
while (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
1145-
{
1146-
sb.Append(Source[Index++]);
1147-
}
1144+
sb.Append(Source[Index++]);
1145+
this.ScanLiteralPart(sb, Character.IsDecimalDigit);
11481146

11491147
ch = Source.CharCodeAt(Index);
11501148
}
@@ -1161,15 +1159,27 @@ public Token ScanNumericLiteral()
11611159

11621160
if (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
11631161
{
1164-
while (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
1165-
{
1166-
sb.Append(Source[Index++]);
1167-
}
1162+
this.ScanLiteralPart(sb, Character.IsDecimalDigit);
11681163
}
11691164
else
11701165
{
11711166
ThrowUnexpectedToken();
11721167
}
1168+
}
1169+
else if (ch == 'n')
1170+
{
1171+
Index++;
1172+
var bigInt = BigInteger.Parse(sb.ToString());
1173+
return new Token
1174+
{
1175+
Type = TokenType.BigIntLiteral,
1176+
Value = bigInt,
1177+
BigIntValue = bigInt,
1178+
LineNumber = LineNumber,
1179+
LineStart = LineStart,
1180+
Start = start,
1181+
End = Index
1182+
};
11731183
}
11741184

11751185
if (Character.IsIdentifierStart(Source.CharCodeAt(Index)))

‎src/Esprima/Token.cs‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
using Esprima.Ast;
1+
using System.Numerics;
2+
using Esprima.Ast;
23

34
namespace Esprima
45
{
@@ -13,7 +14,8 @@ public enum TokenType
1314
Punctuator,
1415
StringLiteral,
1516
RegularExpression,
16-
Template
17+
Template,
18+
BigIntLiteral
1719
};
1820

1921
public class Token
@@ -41,6 +43,7 @@ public class Token
4143
public double NumericValue;
4244
public object? Value;
4345
public RegexValue? RegexValue;
46+
public BigInteger? BigIntValue;
4447

4548
public void Clear()
4649
{
@@ -59,6 +62,7 @@ public void Clear()
5962
NumericValue = 0;
6063
Value = null;
6164
RegexValue = null;
65+
BigIntValue = null;
6266
}
6367
}
6468
}

0 commit comments

Comments
 (0)