diff --git a/src/Esprima/Ast/BigIntLiteral.cs b/src/Esprima/Ast/BigIntLiteral.cs new file mode 100644 index 00000000..2007bcee --- /dev/null +++ b/src/Esprima/Ast/BigIntLiteral.cs @@ -0,0 +1,16 @@ +using System.Numerics; + +namespace Esprima.Ast +{ + public sealed class BigIntLiteral : Literal + { + public readonly string BigInt; + + public BigInteger? BigIntValue => (BigInteger?) Value; + + public BigIntLiteral(BigInteger value, string raw) : base(TokenType.BigIntLiteral, value, raw) + { + BigInt = raw; + } + } +} diff --git a/src/Esprima/Ast/Literal.cs b/src/Esprima/Ast/Literal.cs index b2dbbeca..ebad89c7 100644 --- a/src/Esprima/Ast/Literal.cs +++ b/src/Esprima/Ast/Literal.cs @@ -1,14 +1,16 @@ -using System.Text.RegularExpressions; +using System.Numerics; +using System.Text.RegularExpressions; using Esprima.Utils; namespace Esprima.Ast { - public sealed class Literal : Expression + public class Literal : Expression { public string? StringValue => TokenType == TokenType.StringLiteral ? Value as string : null; public readonly double NumericValue; public bool BooleanValue => TokenType == TokenType.BooleanLiteral && NumericValue != 0; public Regex? RegexValue => TokenType == TokenType.RegularExpression ? (Regex?) Value : null; + public BigInteger? BigIntValue => TokenType == TokenType.BigIntLiteral ? (BigInteger?) Value : null; public readonly RegexValue? Regex; public readonly object? Value; diff --git a/src/Esprima/Character.cs b/src/Esprima/Character.cs index 562782e2..b8cc79c7 100644 --- a/src/Esprima/Character.cs +++ b/src/Esprima/Character.cs @@ -100,7 +100,7 @@ public static bool IsHexDigit(char cp) return cp >= '0' && cp <= '9' || cp >= 'A' && cp <= 'F' || cp >= 'a' && cp <= 'f'; - } + } public static bool IsOctalDigit(char cp) { diff --git a/src/Esprima/JavascriptParser.cs b/src/Esprima/JavascriptParser.cs index ae7b7087..b32cc761 100644 --- a/src/Esprima/JavascriptParser.cs +++ b/src/Esprima/JavascriptParser.cs @@ -590,6 +590,19 @@ private Expression ParsePrimaryExpression() token = NextToken(); raw = GetTokenRaw(token); expr = Finalize(node, new Literal(token.NumericValue, raw)); + break; + + case TokenType.BigIntLiteral: + if (_context.Strict && _lookahead.Octal) + { + TolerateUnexpectedToken(_lookahead, Messages.StrictOctalLiteral); + } + + _context.IsAssignmentTarget = false; + _context.IsBindingElement = false; + token = NextToken(); + raw = GetTokenRaw(token); + expr = Finalize(node, new BigIntLiteral(token.BigIntValue.Value, raw)); break; case TokenType.BooleanLiteral: @@ -860,6 +873,16 @@ private Expression ParseObjectPropertyKey() raw = GetTokenRaw(token); key = Finalize(node, new Literal(token.NumericValue, raw)); + break; + + case TokenType.BigIntLiteral: + if (_context.Strict && token.Octal) + { + TolerateUnexpectedToken(token, Messages.StrictOctalLiteral); + } + + raw = GetTokenRaw(token); + key = Finalize(node, new BigIntLiteral(token.BigIntValue.Value, raw)); break; case TokenType.Identifier: diff --git a/src/Esprima/Messages.cs b/src/Esprima/Messages.cs index 71d06005..dcd8ee9e 100644 --- a/src/Esprima/Messages.cs +++ b/src/Esprima/Messages.cs @@ -40,7 +40,9 @@ public static class Messages public const string MultipleDefaultsInSwitch = "More than one default clause in switch statement"; public const string NewlineAfterThrow = "Illegal newline after throw"; public const string NoAsAfterImportNamespace = "Unexpected token"; - public const string NoCatchOrFinally = "Missing catch or finally after try"; + public const string NoCatchOrFinally = "Missing catch or finally after try"; + public const string NumericSeperatorOneUnderscore = "Numeric separator must be exactly one underscore"; + public const string NumericSeperatorNotAllowedHere = "Numeric separator is not allowed here"; public const string ParameterAfterRestParameter = "Rest parameter must be last formal parameter"; public const string PropertyAfterRestProperty = "Unexpected token"; public const string Redeclaration = "{0} \"{1}\" has already been declared"; diff --git a/src/Esprima/Scanner.cs b/src/Esprima/Scanner.cs index 0bf16811..5fca05e0 100644 --- a/src/Esprima/Scanner.cs +++ b/src/Esprima/Scanner.cs @@ -1,6 +1,7 @@ using System; using System.Collections.Generic; using System.Globalization; +using System.Numerics; using System.Runtime.CompilerServices; using System.Text; using System.Text.RegularExpressions; @@ -887,20 +888,10 @@ static string SafeSubstring(string s, int startIndex, int length) // https://tc39.github.io/ecma262/#sec-literals-numeric-literals public Token ScanHexLiteral(int start) - { - var index = Index; - - while (!Eof()) - { - if (!Character.IsHexDigit(Source.CharCodeAt(Index))) - { - break; - } - - Index++; - } - - var number = Source.Substring(index, Index - index); + { + var sb = GetStringBuilder(); + this.ScanLiteralPart(sb, Character.IsHexDigit); + var number = sb.ToString(); if (number.Length == 0) { @@ -957,22 +948,11 @@ public Token ScanHexLiteral(int start) } public Token ScanBinaryLiteral(int start) - { + { char ch; - var index = Index; - - while (!Eof()) - { - ch = Source[Index]; - if (ch != '0' && ch != '1') - { - break; - } - - Index++; - } - - var number = Source.Substring(index, Index - index); + var sb = GetStringBuilder(); + this.ScanLiteralPart(sb, c => c == '0' || c == '1'); + var number = sb.ToString(); if (number.Length == 0) { @@ -1015,18 +995,9 @@ public Token ScanOctalLiteral(char prefix, int start) else { ++Index; - } - - while (!Eof()) - { - if (!Character.IsOctalDigit(Source.CharCodeAt(Index))) - { - break; - } - - sb.Append(Source[Index++]); - } - + } + + this.ScanLiteralPart(sb, Character.IsOctalDigit); var number = sb.ToString(); if (!octal && number.Length == 0) @@ -1084,6 +1055,40 @@ public bool IsImplicitOctalLiteral() return true; } + private void ScanLiteralPart(StringBuilder sb, Func check) + { + var charCode = Source.CharCodeAt(Index); + if (charCode == '_') + { + ThrowUnexpectedToken(Messages.NumericSeperatorNotAllowedHere); + } + + while ((check(charCode) || charCode == '_')) + { + if (charCode != '_') + { + sb.Append(charCode); + } + Index++; + var newCharCode = Source.CharCodeAt(Index); + if (charCode == '_' && newCharCode == '_') + { + ThrowUnexpectedToken(Messages.NumericSeperatorOneUnderscore); + } + + if (Eof()) + { + break; + } + charCode = newCharCode; + } + + if (charCode == '_') + { + ThrowUnexpectedToken(Messages.NumericSeperatorNotAllowedHere); + } + } + public Token ScanNumericLiteral() { var sb = GetStringBuilder(); @@ -1095,7 +1100,6 @@ public Token ScanNumericLiteral() if (ch != '.') { var first = Source[Index++]; - sb.Append(first); ch = Source.CharCodeAt(Index); // Hex number starts with '0x'. @@ -1130,21 +1134,15 @@ public Token ScanNumericLiteral() } } - while (Character.IsDecimalDigit(Source.CharCodeAt(Index))) - { - sb.Append(Source[Index++]); - } - + --Index; + this.ScanLiteralPart(sb, Character.IsDecimalDigit); ch = Source.CharCodeAt(Index); } if (ch == '.') { - sb.Append(Source[Index++]); - while (Character.IsDecimalDigit(Source.CharCodeAt(Index))) - { - sb.Append(Source[Index++]); - } + sb.Append(Source[Index++]); + this.ScanLiteralPart(sb, Character.IsDecimalDigit); ch = Source.CharCodeAt(Index); } @@ -1161,15 +1159,27 @@ public Token ScanNumericLiteral() if (Character.IsDecimalDigit(Source.CharCodeAt(Index))) { - while (Character.IsDecimalDigit(Source.CharCodeAt(Index))) - { - sb.Append(Source[Index++]); - } + this.ScanLiteralPart(sb, Character.IsDecimalDigit); } else { ThrowUnexpectedToken(); } + } + else if (ch == 'n') + { + Index++; + var bigInt = BigInteger.Parse(sb.ToString()); + return new Token + { + Type = TokenType.BigIntLiteral, + Value = bigInt, + BigIntValue = bigInt, + LineNumber = LineNumber, + LineStart = LineStart, + Start = start, + End = Index + }; } if (Character.IsIdentifierStart(Source.CharCodeAt(Index))) diff --git a/src/Esprima/Token.cs b/src/Esprima/Token.cs index 4ddfd6d3..49048321 100644 --- a/src/Esprima/Token.cs +++ b/src/Esprima/Token.cs @@ -1,4 +1,5 @@ -using Esprima.Ast; +using System.Numerics; +using Esprima.Ast; namespace Esprima { @@ -13,7 +14,8 @@ public enum TokenType Punctuator, StringLiteral, RegularExpression, - Template + Template, + BigIntLiteral }; public class Token @@ -41,6 +43,7 @@ public class Token public double NumericValue; public object? Value; public RegexValue? RegexValue; + public BigInteger? BigIntValue; public void Clear() { @@ -59,6 +62,7 @@ public void Clear() NumericValue = 0; Value = null; RegexValue = null; + BigIntValue = null; } } } diff --git a/test/Esprima.Tests/Fixtures.cs b/test/Esprima.Tests/Fixtures.cs index 85e73baf..06cfd0b2 100644 --- a/test/Esprima.Tests/Fixtures.cs +++ b/test/Esprima.Tests/Fixtures.cs @@ -1,219 +1,219 @@ -using System; -using System.Collections.Generic; -using System.IO; -using System.Linq; -using System.Reflection; -using Esprima.Ast; -using Esprima.Utils; -using Newtonsoft.Json.Linq; -using Xunit; - -namespace Esprima.Test -{ - public class Fixtures - { - // Do manually set it to true to update local test files with the current results. - // Only use this when the test is deemed wrong. - const bool WriteBackExpectedTree = false; - - [Fact] - public void HoistingScopeShouldWork() - { - var parser = new JavaScriptParser(@" - function p() {} - var x;"); - var program = parser.ParseScript(); - } - - private static string ParseAndFormat(SourceType sourceType, string source, ParserOptions options) - { - var parser = new JavaScriptParser(source, options); - var program = sourceType == SourceType.Script ? (Program) parser.ParseScript() : parser.ParseModule(); - const string indent = " "; - return program.ToJsonString( - AstJson.Options.Default - .WithIncludingLineColumn(true) - .WithIncludingRange(true), - indent - ); - } - - private static bool CompareTreesInternal(string actual, string expected) - { - var actualJObject = JObject.Parse(actual); - var expectedJObject = JObject.Parse(expected); - - // Don't compare the tokens array as it's not in the generated AST - expectedJObject.Remove("tokens"); - expectedJObject.Remove("comments"); - expectedJObject.Remove("errors"); - - return JToken.DeepEquals(actualJObject, expectedJObject); - } - - private static void CompareTrees(string actual, string expected, string path) - { - var actualJObject = JObject.Parse(actual); - var expectedJObject = JObject.Parse(expected); - - // Don't compare the tokens array as it's not in the generated AST - expectedJObject.Remove("tokens"); - expectedJObject.Remove("comments"); - expectedJObject.Remove("errors"); - - var areEqual = JToken.DeepEquals(actualJObject, expectedJObject); - if (!areEqual) - { - var actualString = actualJObject.ToString(); - var expectedString = expectedJObject.ToString(); - Assert.Equal(expectedString, actualString); - } - } - - [Theory] - [MemberData(nameof(SourceFiles), "Fixtures")] - public void ExecuteTestCase(string fixture) - { - - var options = new ParserOptions { Tokens = true }; - - string treeFilePath, failureFilePath, moduleFilePath; - var jsFilePath = Path.Combine(GetFixturesPath(), "Fixtures", fixture); - if (jsFilePath.EndsWith(".source.js")) - { - treeFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(jsFilePath))) + ".tree.json"; - failureFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(jsFilePath))) + ".failure.json"; - moduleFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(jsFilePath))) + ".module.json"; - } - else - { - treeFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(jsFilePath)) + ".tree.json"; - failureFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(jsFilePath)) + ".failure.json"; - moduleFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(jsFilePath)) + ".module.json"; - } - - var script = File.ReadAllText(jsFilePath); - if (jsFilePath.EndsWith(".source.js")) - { - var parser = new JavaScriptParser(script); - var program = parser.ParseScript(); - var source = program.Body.First().As().Declarations.First().As().Init.As().StringValue; - script = source; - } - - var expected = ""; - var invalid = false; - - var filename = Path.GetFileNameWithoutExtension(jsFilePath); - - var isModule = - filename.Contains("module") || - filename.Contains("export") || - filename.Contains("import"); - - if (!filename.Contains(".module")) - { - isModule &= !jsFilePath.Contains("dynamic-import") && !jsFilePath.Contains("script"); - } - - var sourceType = isModule - ? SourceType.Module - : SourceType.Script; - -#pragma warning disable 162 - if (File.Exists(moduleFilePath)) - { - sourceType = SourceType.Module; - expected = File.ReadAllText(moduleFilePath); - if (WriteBackExpectedTree) - { - var actual = ParseAndFormat(sourceType, script, options); - if (!CompareTreesInternal(actual, expected)) - File.WriteAllText(moduleFilePath, actual); - } - } - else if (File.Exists(treeFilePath)) - { - expected = File.ReadAllText(treeFilePath); - if (WriteBackExpectedTree) - - { - var actual = ParseAndFormat(sourceType, script, options); - if (!CompareTreesInternal(actual, expected)) - File.WriteAllText(treeFilePath, actual); - } - } - else if (File.Exists(failureFilePath)) - { - invalid = true; - expected = File.ReadAllText(failureFilePath); - if (WriteBackExpectedTree) - { - var actual = ParseAndFormat(sourceType, script, options); - if (!CompareTreesInternal(actual, expected)) - File.WriteAllText(failureFilePath, actual); - } -#pragma warning restore 162 - } - else - { - // cannot compare - return; - } - - invalid |= - filename.Contains("error") || - filename.Contains("invalid") && !filename.Contains("invalid-yield-object-"); - - if (!invalid) - { - options.Tolerant = true; - - var actual = ParseAndFormat(sourceType, script, options); - CompareTrees(actual, expected, jsFilePath); - } - else - { - options.Tolerant = false; - - // TODO: check the accuracy of the message and of the location - Assert.Throws(() => ParseAndFormat(sourceType, script, options)); - } - } - - public static IEnumerable SourceFiles(string relativePath) - { - var fixturesPath = Path.Combine(GetFixturesPath(), relativePath); - - var files = Directory.GetFiles(fixturesPath, "*.js", SearchOption.AllDirectories); - - return files - .Select(x => new object[] { x.Substring(fixturesPath.Length + 1) }) - .ToList(); - } - - internal static string GetFixturesPath() - { -#if NET461 - var assemblyPath = new Uri(typeof(Fixtures).GetTypeInfo().Assembly.CodeBase).LocalPath; - var assemblyDirectory = new FileInfo(assemblyPath).Directory; -#else - var assemblyPath = typeof(Fixtures).GetTypeInfo().Assembly.Location; - var assemblyDirectory = new FileInfo(assemblyPath).Directory; -#endif - var root = assemblyDirectory.Parent.Parent.Parent.FullName; - return root; - } - - [Fact] - public void CommentsAreParsed() - { - var count = 0; - Action action = node => count++; - var parser = new JavaScriptParser("// this is a comment", new ParserOptions(), action); - parser.ParseScript(); - - Assert.Equal(1, count); - } - } -} +using System; +using System.Collections.Generic; +using System.IO; +using System.Linq; +using System.Reflection; +using Esprima.Ast; +using Esprima.Utils; +using Newtonsoft.Json.Linq; +using Xunit; + +namespace Esprima.Test +{ + public class Fixtures + { + // Do manually set it to true to update local test files with the current results. + // Only use this when the test is deemed wrong. + const bool WriteBackExpectedTree = false; + + [Fact] + public void HoistingScopeShouldWork() + { + var parser = new JavaScriptParser(@" + function p() {} + var x;"); + var program = parser.ParseScript(); + } + + private static string ParseAndFormat(SourceType sourceType, string source, ParserOptions options) + { + var parser = new JavaScriptParser(source, options); + var program = sourceType == SourceType.Script ? (Program) parser.ParseScript() : parser.ParseModule(); + const string indent = " "; + return program.ToJsonString( + AstJson.Options.Default + .WithIncludingLineColumn(true) + .WithIncludingRange(true), + indent + ); + } + + private static bool CompareTreesInternal(string actual, string expected) + { + var actualJObject = JObject.Parse(actual); + var expectedJObject = JObject.Parse(expected); + + // Don't compare the tokens array as it's not in the generated AST + expectedJObject.Remove("tokens"); + expectedJObject.Remove("comments"); + expectedJObject.Remove("errors"); + + return JToken.DeepEquals(actualJObject, expectedJObject); + } + + private static void CompareTrees(string actual, string expected, string path) + { + var actualJObject = JObject.Parse(actual); + var expectedJObject = JObject.Parse(expected); + + // Don't compare the tokens array as it's not in the generated AST + expectedJObject.Remove("tokens"); + expectedJObject.Remove("comments"); + expectedJObject.Remove("errors"); + + var areEqual = JToken.DeepEquals(actualJObject, expectedJObject); + if (!areEqual) + { + var actualString = actualJObject.ToString(); + var expectedString = expectedJObject.ToString(); + Assert.Equal(expectedString, actualString); + } + } + + [Theory] + [MemberData(nameof(SourceFiles), "Fixtures")] + public void ExecuteTestCase(string fixture) + { + + var options = new ParserOptions { Tokens = true }; + + string treeFilePath, failureFilePath, moduleFilePath; + var jsFilePath = Path.Combine(GetFixturesPath(), "Fixtures", fixture); + if (jsFilePath.EndsWith(".source.js")) + { + treeFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(jsFilePath))) + ".tree.json"; + failureFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(jsFilePath))) + ".failure.json"; + moduleFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(Path.GetFileNameWithoutExtension(jsFilePath))) + ".module.json"; + } + else + { + treeFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(jsFilePath)) + ".tree.json"; + failureFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(jsFilePath)) + ".failure.json"; + moduleFilePath = Path.Combine(Path.GetDirectoryName(jsFilePath), Path.GetFileNameWithoutExtension(jsFilePath)) + ".module.json"; + } + + var script = File.ReadAllText(jsFilePath); + if (jsFilePath.EndsWith(".source.js")) + { + var parser = new JavaScriptParser(script); + var program = parser.ParseScript(); + var source = program.Body.First().As().Declarations.First().As().Init.As().StringValue; + script = source; + } + + var expected = ""; + var invalid = false; + + var filename = Path.GetFileNameWithoutExtension(jsFilePath); + + var isModule = + filename.Contains("module") || + filename.Contains("export") || + filename.Contains("import"); + + if (!filename.Contains(".module")) + { + isModule &= !jsFilePath.Contains("dynamic-import") && !jsFilePath.Contains("script"); + } + + var sourceType = isModule + ? SourceType.Module + : SourceType.Script; + +#pragma warning disable 162 + if (File.Exists(moduleFilePath)) + { + sourceType = SourceType.Module; + expected = File.ReadAllText(moduleFilePath); + if (WriteBackExpectedTree) + { + var actual = ParseAndFormat(sourceType, script, options); + if (!CompareTreesInternal(actual, expected)) + File.WriteAllText(moduleFilePath, actual); + } + } + else if (File.Exists(treeFilePath)) + { + expected = File.ReadAllText(treeFilePath); + if (WriteBackExpectedTree) + + { + var actual = ParseAndFormat(sourceType, script, options); + if (!CompareTreesInternal(actual, expected)) + File.WriteAllText(treeFilePath, actual); + } + } + else if (File.Exists(failureFilePath)) + { + invalid = true; + expected = File.ReadAllText(failureFilePath); + if (WriteBackExpectedTree) + { + var actual = ParseAndFormat(sourceType, script, options); + if (!CompareTreesInternal(actual, expected)) + File.WriteAllText(failureFilePath, actual); + } +#pragma warning restore 162 + } + else + { + // cannot compare + return; + } + + invalid |= + filename.Contains("error") || + filename.Contains("invalid") && !filename.Contains("invalid-yield-object-"); + + if (!invalid) + { + options.Tolerant = true; + + var actual = ParseAndFormat(sourceType, script, options); + CompareTrees(actual, expected, jsFilePath); + } + else + { + options.Tolerant = false; + + // TODO: check the accuracy of the message and of the location + Assert.Throws(() => ParseAndFormat(sourceType, script, options)); + } + } + + public static IEnumerable SourceFiles(string relativePath) + { + var fixturesPath = Path.Combine(GetFixturesPath(), relativePath); + + var files = Directory.GetFiles(fixturesPath, "*.js", SearchOption.AllDirectories); + + return files + .Select(x => new object[] { x.Substring(fixturesPath.Length + 1) }) + .ToList(); + } + + internal static string GetFixturesPath() + { +#if NET461 + var assemblyPath = new Uri(typeof(Fixtures).GetTypeInfo().Assembly.CodeBase).LocalPath; + var assemblyDirectory = new FileInfo(assemblyPath).Directory; +#else + var assemblyPath = typeof(Fixtures).GetTypeInfo().Assembly.Location; + var assemblyDirectory = new FileInfo(assemblyPath).Directory; +#endif + var root = assemblyDirectory.Parent.Parent.Parent.FullName; + return root; + } + + [Fact] + public void CommentsAreParsed() + { + var count = 0; + Action action = node => count++; + var parser = new JavaScriptParser("// this is a comment", new ParserOptions(), action); + parser.ParseScript(); + + Assert.Equal(1, count); + } + } +} diff --git a/test/Esprima.Tests/SeparatorTests.cs b/test/Esprima.Tests/SeparatorTests.cs new file mode 100644 index 00000000..fa8084a6 --- /dev/null +++ b/test/Esprima.Tests/SeparatorTests.cs @@ -0,0 +1,27 @@ +using Esprima.Ast; +using Xunit; + +namespace Esprima.Tests +{ + public class SeparatorTests + { + [Fact] + public void CanParseSeperators() + { + var script = new JavaScriptParser("var foo=12_3_456").ParseScript(); + Assert.Equal(123456, (double) ((Literal) ((VariableDeclaration) script.Body[0]).Declarations[0].Init).Value); + } + + [Fact] + public void Fails1() + { + Assert.Throws(() => new JavaScriptParser("var foo=12__3_456").ParseScript()); + } + + [Fact] + public void Fails2() + { + Assert.Throws(() => new JavaScriptParser("var foo=12_3_456_").ParseScript()); + } + } +}