BigInt & numeric separator support

src/Esprima/Ast/BigIntLiteral.cs

@@ -0,0 +1,16 @@
+using System.Numerics;
+
+namespace Esprima.Ast
+{
+    public sealed class BigIntLiteral : Literal
+    {
+        public readonly string BigInt;
+
+        public BigInteger? BigIntValue => (BigInteger?) Value;
+
+        public BigIntLiteral(BigInteger value, string raw) : base(TokenType.BigIntLiteral, value, raw)
+        {
+            BigInt = raw;
+        }
+    }
+}

src/Esprima/Ast/Literal.cs

@@ -1,14 +1,16 @@
+using System.Numerics;
 using System.Text.RegularExpressions;
 using Esprima.Utils;
 
 namespace Esprima.Ast
 {
-    public sealed class Literal : Expression
+    public class Literal : Expression
     {
         public string? StringValue => TokenType == TokenType.StringLiteral ? Value as string : null;
         public readonly double NumericValue;
         public bool BooleanValue => TokenType == TokenType.BooleanLiteral && NumericValue != 0;
         public Regex? RegexValue => TokenType == TokenType.RegularExpression ? (Regex?) Value : null;
+        public BigInteger? BigIntValue => TokenType == TokenType.BigIntLiteral ? (BigInteger?) Value : null;
 
         public readonly RegexValue? Regex;
         public readonly object? Value;

src/Esprima/Character.cs

@@ -100,7 +100,7 @@ public static bool IsHexDigit(char cp)
             return cp >= '0' && cp <= '9' ||
                    cp >= 'A' && cp <= 'F' ||
                    cp >= 'a' && cp <= 'f';
-        }
+        }
 
         public static bool IsOctalDigit(char cp)
         {

src/Esprima/JavascriptParser.cs

@@ -589,6 +589,19 @@ private Expression ParsePrimaryExpression()
                     token = NextToken();
                     raw = GetTokenRaw(token);
                     expr = Finalize(node, new Literal(token.NumericValue, raw));
+                    break;
+
+                case TokenType.BigIntLiteral:
+                    if (_context.Strict && _lookahead.Octal)
+                    {
+                        TolerateUnexpectedToken(_lookahead, Messages.StrictOctalLiteral);
+                    }
+
+                    _context.IsAssignmentTarget = false;
+                    _context.IsBindingElement = false;
+                    token = NextToken();
+                    raw = GetTokenRaw(token);
+                    expr = Finalize(node, new BigIntLiteral(token.BigIntValue.Value, raw));
                     break;
 
                 case TokenType.BooleanLiteral:
@@ -859,6 +872,16 @@ private Expression ParseObjectPropertyKey()
 
                     raw = GetTokenRaw(token);
                     key = Finalize(node, new Literal(token.NumericValue, raw));
+                    break;
+
+                case TokenType.BigIntLiteral:
+                    if (_context.Strict && token.Octal)
+                    {
+                        TolerateUnexpectedToken(token, Messages.StrictOctalLiteral);
+                    }
+
+                    raw = GetTokenRaw(token);
+                    key = Finalize(node, new BigIntLiteral(token.BigIntValue.Value, raw));
                     break;
 
                 case TokenType.Identifier:

src/Esprima/Messages.cs

@@ -40,7 +40,9 @@ public static class Messages
         public const string MultipleDefaultsInSwitch = "More than one default clause in switch statement";
         public const string NewlineAfterThrow = "Illegal newline after throw";
         public const string NoAsAfterImportNamespace = "Unexpected token";
-        public const string NoCatchOrFinally = "Missing catch or finally after try";
+        public const string NoCatchOrFinally = "Missing catch or finally after try";
+        public const string NumericSeperatorOneUnderscore = "Numeric separator must be exactly one underscore";
+        public const string NumericSeperatorNotAllowedHere = "Numeric separator is not allowed here";
         public const string ParameterAfterRestParameter = "Rest parameter must be last formal parameter";
         public const string PropertyAfterRestProperty = "Unexpected token";
         public const string Redeclaration = "{0} \"{1}\" has already been declared";

src/Esprima/Scanner.cs

@@ -1,6 +1,7 @@
 using System;
 using System.Collections.Generic;
 using System.Globalization;
+using System.Numerics;
 using System.Runtime.CompilerServices;
 using System.Text;
 using System.Text.RegularExpressions;
@@ -888,19 +889,7 @@ static string SafeSubstring(string s, int startIndex, int length)
 
         public Token ScanHexLiteral(int start)
         {
-            var index = Index;
-
-            while (!Eof())
-            {
-                if (!Character.IsHexDigit(Source.CharCodeAt(Index)))
-                {
-                    break;
-                }
-
-                Index++;
-            }
-
-            var number = Source.Substring(index, Index - index);
+            var number = this.ScanLiteralPart(Character.IsHexDigit);
 
             if (number.Length == 0)
             {
@@ -957,23 +946,10 @@ public Token ScanHexLiteral(int start)
         }
 
         public Token ScanBinaryLiteral(int start)
-        {
+        {
             char ch;
-            var index = Index;
-
-            while (!Eof())
-            {
-                ch = Source[Index];
-                if (ch != '0' && ch != '1')
-                {
-                    break;
-                }
-
-                Index++;
-            }
-
-            var number = Source.Substring(index, Index - index);
-
+            var number = this.ScanLiteralPart(c => c == '0' || c == '1');
+
             if (number.Length == 0)
             {
                 // only 0b or 0B
@@ -1015,18 +991,9 @@ public Token ScanOctalLiteral(char prefix, int start)
             else
             {
                 ++Index;
-            }
-
-            while (!Eof())
-            {
-                if (!Character.IsOctalDigit(Source.CharCodeAt(Index)))
-                {
-                    break;
-                }
-
-                sb.Append(Source[Index++]);
-            }
-
+            }
+
+            sb.Append(this.ScanLiteralPart(Character.IsOctalDigit));
             var number = sb.ToString();
 
             if (!octal && number.Length == 0)
@@ -1084,6 +1051,27 @@ public bool IsImplicitOctalLiteral()
             return true;
         }
 
+        private string ScanLiteralPart(Func<char, bool> check)
+        {
+            string num = "";
+
+            if (Source.CharCodeAt(Index) == '_')
+                ThrowUnexpectedToken(Messages.NumericSeperatorNotAllowedHere);
+
+            while (!Eof() && (check(Source.CharCodeAt(Index)) || Source.CharCodeAt(Index) == '_')) {
+                if (Source.CharCodeAt(Index) != '_')
+                    num += Source.CharCodeAt(Index);
+                Index++;
+                if (Source.CharCodeAt(Index-1) == '_' && Source.CharCodeAt(Index) == '_')
+                    ThrowUnexpectedToken(Messages.NumericSeperatorOneUnderscore);
+            }
+
+            if (Source.CharCodeAt(Index-1) == '_')
+                ThrowUnexpectedToken(Messages.NumericSeperatorNotAllowedHere);
+
+            return num;
+        }
+
         public Token ScanNumericLiteral()
         {
             var sb = GetStringBuilder();
@@ -1095,7 +1083,6 @@ public Token ScanNumericLiteral()
             if (ch != '.')
             {
                 var first = Source[Index++];
-                sb.Append(first);
                 ch = Source.CharCodeAt(Index);
 
                 // Hex number starts with '0x'.
@@ -1130,21 +1117,15 @@ public Token ScanNumericLiteral()
                     }
                 }
 
-                while (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
-                {
-                    sb.Append(Source[Index++]);
-                }
-
+                --Index;
+                sb.Append(this.ScanLiteralPart(Character.IsDecimalDigit));
                 ch = Source.CharCodeAt(Index);
             }
 
             if (ch == '.')
             {
-                sb.Append(Source[Index++]);
-                while (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
-                {
-                    sb.Append(Source[Index++]);
-                }
+                sb.Append(Source[Index++]);
+                sb.Append(this.ScanLiteralPart(Character.IsDecimalDigit));
 
                 ch = Source.CharCodeAt(Index);
             }
@@ -1161,15 +1142,27 @@ public Token ScanNumericLiteral()
 
                 if (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
                 {
-                    while (Character.IsDecimalDigit(Source.CharCodeAt(Index)))
-                    {
-                        sb.Append(Source[Index++]);
-                    }
+                    sb.Append(this.ScanLiteralPart(Character.IsDecimalDigit));
                 }
                 else
                 {
                     ThrowUnexpectedToken();
                 }
+            }
+            else if (ch == 'n')
+            {
+                Index++;
+                var bigInt = BigInteger.Parse(sb.ToString());
+                return new Token
+                {
+                    Type = TokenType.BigIntLiteral,
+                    Value = bigInt,
+                    BigIntValue = bigInt,
+                    LineNumber = LineNumber,
+                    LineStart = LineStart,
+                    Start = start,
+                    End = Index
+                };
             }
 
             if (Character.IsIdentifierStart(Source.CharCodeAt(Index)))

src/Esprima/Token.cs

@@ -1,4 +1,5 @@
-using Esprima.Ast;
+using System.Numerics;
+using Esprima.Ast;
 
 namespace Esprima
 {
@@ -13,7 +14,8 @@ public enum TokenType
         Punctuator,
         StringLiteral,
         RegularExpression,
-        Template
+        Template,
+        BigIntLiteral
     };
 
     public class Token
@@ -41,6 +43,7 @@ public class Token
         public double NumericValue;
         public object? Value;
         public RegexValue? RegexValue;
+        public BigInteger? BigIntValue;
 
         public void Clear()
         {
@@ -59,6 +62,7 @@ public void Clear()
             NumericValue = 0;
             Value = null;
             RegexValue = null;
+            BigIntValue = null;
         }
     }
 }