Skip to content

BigInt & numeric separator support #204

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
lahma merged 5 commits into from
Nov 4, 2021
Merged

BigInt & numeric separator support #204

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
a1784dc
bigint & numeric seperator support
jogibear9988 Nov 1, 2021
0a57267
perfomance optimations and a test
jogibear9988 Nov 2, 2021
18307e2
fix brakets
jogibear9988 Nov 2, 2021
2fbe694
Merge branch 'main' into fixBigIntAndSep
sebastienros Nov 2, 2021
3cf3080
Update Literal.cs
sebastienros Nov 2, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions src/Esprima/Ast/BigIntLiteral.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
using System.Numerics;

namespace Esprima.Ast
{
public sealed class BigIntLiteral : Literal
{
public readonly string BigInt;

public BigInteger? BigIntValue => (BigInteger?) Value;
Copy link
Collaborator

@lahma lahma Nov 2, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

do we really need to add this as Literal already has that, I think it should be quite cheap to call even with the type check

Copy link
Contributor Author

@jogibear9988 jogibear9988 Nov 2, 2021

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I did it to match the Tree spec: https://github.com/estree/estree/blob/master/es2020.md#bigintliteral

Copy link
Contributor Author

@jogibear9988 jogibear9988 Nov 2, 2021 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Cause I think the AST should be nearly the same in all Javascript Parsers


public BigIntLiteral(BigInteger value, string raw) : base(TokenType.BigIntLiteral, value, raw)
{
BigInt = raw;
}
}
}
6 changes: 4 additions & 2 deletions src/Esprima/Ast/Literal.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,14 +1,16 @@
using System.Text.RegularExpressions;
using System.Numerics;
using System.Text.RegularExpressions;
using Esprima.Utils;

namespace Esprima.Ast
{
public sealed class Literal : Expression
public class Literal : Expression
{
public string? StringValue => TokenType == TokenType.StringLiteral ? Value as string : null;
public readonly double NumericValue;
public bool BooleanValue => TokenType == TokenType.BooleanLiteral && NumericValue != 0;
public Regex? RegexValue => TokenType == TokenType.RegularExpression ? (Regex?) Value : null;
public BigInteger? BigIntValue => TokenType == TokenType.BigIntLiteral ? (BigInteger?) Value : null;

public readonly RegexValue? Regex;
public readonly object? Value;
Expand Down
2 changes: 1 addition & 1 deletion src/Esprima/Character.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -100,7 +100,7 @@ public static bool IsHexDigit(char cp)
return cp >= '0' && cp <= '9' ||
cp >= 'A' && cp <= 'F' ||
cp >= 'a' && cp <= 'f';
}
}

public static bool IsOctalDigit(char cp)
{
Expand Down
23 changes: 23 additions & 0 deletions src/Esprima/JavascriptParser.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,6 +590,19 @@ private Expression ParsePrimaryExpression()
token = NextToken();
raw = GetTokenRaw(token);
expr = Finalize(node, new Literal(token.NumericValue, raw));
break;

case TokenType.BigIntLiteral:
if (_context.Strict && _lookahead.Octal)
{
TolerateUnexpectedToken(_lookahead, Messages.StrictOctalLiteral);
}

_context.IsAssignmentTarget = false;
_context.IsBindingElement = false;
token = NextToken();
raw = GetTokenRaw(token);
expr = Finalize(node, new BigIntLiteral(token.BigIntValue.Value, raw));
break;

case TokenType.BooleanLiteral:
Expand Down Expand Up @@ -860,6 +873,16 @@ private Expression ParseObjectPropertyKey()

raw = GetTokenRaw(token);
key = Finalize(node, new Literal(token.NumericValue, raw));
break;

case TokenType.BigIntLiteral:
if (_context.Strict && token.Octal)
{
TolerateUnexpectedToken(token, Messages.StrictOctalLiteral);
}

raw = GetTokenRaw(token);
key = Finalize(node, new BigIntLiteral(token.BigIntValue.Value, raw));
break;

case TokenType.Identifier:
Expand Down
4 changes: 3 additions & 1 deletion src/Esprima/Messages.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -40,7 +40,9 @@ public static class Messages
public const string MultipleDefaultsInSwitch = "More than one default clause in switch statement";
public const string NewlineAfterThrow = "Illegal newline after throw";
public const string NoAsAfterImportNamespace = "Unexpected token";
public const string NoCatchOrFinally = "Missing catch or finally after try";
public const string NoCatchOrFinally = "Missing catch or finally after try";
public const string NumericSeperatorOneUnderscore = "Numeric separator must be exactly one underscore";
public const string NumericSeperatorNotAllowedHere = "Numeric separator is not allowed here";
public const string ParameterAfterRestParameter = "Rest parameter must be last formal parameter";
public const string PropertyAfterRestProperty = "Unexpected token";
public const string Redeclaration = "{0} \"{1}\" has already been declared";
Expand Down
122 changes: 66 additions & 56 deletions src/Esprima/Scanner.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
using System;
using System.Collections.Generic;
using System.Globalization;
using System.Numerics;
using System.Runtime.CompilerServices;
using System.Text;
using System.Text.RegularExpressions;
Expand Down Expand Up @@ -887,20 +888,10 @@ static string SafeSubstring(string s, int startIndex, int length)
// https://tc39.github.io/ecma262/#sec-literals-numeric-literals

public Token ScanHexLiteral(int start)
{
var index = Index;

while (!Eof())
{
if (!Character.IsHexDigit(Source.CharCodeAt(Index)))
{
break;
}

Index++;
}

var number = Source.Substring(index, Index - index);
{
var sb = GetStringBuilder();
this.ScanLiteralPart(sb, Character.IsHexDigit);
var number = sb.ToString();

if (number.Length == 0)
{
Expand Down Expand Up @@ -957,22 +948,11 @@ public Token ScanHexLiteral(int start)
}

public Token ScanBinaryLiteral(int start)
{
{
char ch;
var index = Index;

while (!Eof())
{
ch = Source[Index];
if (ch != '0' && ch != '1')
{
break;
}

Index++;
}

var number = Source.Substring(index, Index - index);
var sb = GetStringBuilder();
this.ScanLiteralPart(sb, c => c == '0' || c == '1');
var number = sb.ToString();

if (number.Length == 0)
{
Expand Down Expand Up @@ -1015,18 +995,9 @@ public Token ScanOctalLiteral(char prefix, int start)
else
{
++Index;
}

while (!Eof())
{
if (!Character.IsOctalDigit(Source.CharCodeAt(Index)))
{
break;
}

sb.Append(Source[Index++]);
}

}

this.ScanLiteralPart(sb, Character.IsOctalDigit);
var number = sb.ToString();

if (!octal && number.Length == 0)
Expand Down Expand Up @@ -1084,6 +1055,40 @@ public bool IsImplicitOctalLiteral()
return true;
}

private void ScanLiteralPart(StringBuilder sb, Func<char, bool> check)
{
var charCode = Source.CharCodeAt(Index);
if (charCode == '_')
{
ThrowUnexpectedToken(Messages.NumericSeperatorNotAllowedHere);
}

while ((check(charCode) || charCode == '_'))
{
if (charCode != '_')
{
sb.Append(charCode);
}
Index++;
var newCharCode = Source.CharCodeAt(Index);
if (charCode == '_' && newCharCode == '_')
{
ThrowUnexpectedToken(Messages.NumericSeperatorOneUnderscore);
}

if (Eof())
{
break;
}
charCode = newCharCode;
}

if (charCode == '_')
{
ThrowUnexpectedToken(Messages.NumericSeperatorNotAllowedHere);
}
}

public Token ScanNumericLiteral()
{
var sb = GetStringBuilder();
Expand All @@ -1095,7 +1100,6 @@ public Token ScanNumericLiteral()
if (ch != '.')
{
var first = Source[Index++];
sb.Append(first);
ch = Source.CharCodeAt(Index);

// Hex number starts with '0x'.
Expand Down Expand Up @@ -1130,21 +1134,15 @@ public Token ScanNumericLiteral()
}
}

while (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
{
sb.Append(Source[Index++]);
}

--Index;
this.ScanLiteralPart(sb, Character.IsDecimalDigit);
ch = Source.CharCodeAt(Index);
}

if (ch == '.')
{
sb.Append(Source[Index++]);
while (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
{
sb.Append(Source[Index++]);
}
sb.Append(Source[Index++]);
this.ScanLiteralPart(sb, Character.IsDecimalDigit);

ch = Source.CharCodeAt(Index);
}
Expand All @@ -1161,15 +1159,27 @@ public Token ScanNumericLiteral()

if (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
{
while (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
{
sb.Append(Source[Index++]);
}
this.ScanLiteralPart(sb, Character.IsDecimalDigit);
}
else
{
ThrowUnexpectedToken();
}
}
else if (ch == 'n')
{
Index++;
var bigInt = BigInteger.Parse(sb.ToString());
return new Token
{
Type = TokenType.BigIntLiteral,
Value = bigInt,
BigIntValue = bigInt,
LineNumber = LineNumber,
LineStart = LineStart,
Start = start,
End = Index
};
}

if (Character.IsIdentifierStart(Source.CharCodeAt(Index)))
Expand Down
8 changes: 6 additions & 2 deletions src/Esprima/Token.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -1,4 +1,5 @@
using Esprima.Ast;
using System.Numerics;
using Esprima.Ast;

namespace Esprima
{
Expand All @@ -13,7 +14,8 @@ public enum TokenType
Punctuator,
StringLiteral,
RegularExpression,
Template
Template,
BigIntLiteral
};

public class Token
Expand Down Expand Up @@ -41,6 +43,7 @@ public class Token
public double NumericValue;
public object? Value;
public RegexValue? RegexValue;
public BigInteger? BigIntValue;

public void Clear()
{
Expand All @@ -59,6 +62,7 @@ public void Clear()
NumericValue = 0;
Value = null;
RegexValue = null;
BigIntValue = null;
}
}
}
Loading