Implement top-level await in modules feature (#361)

adams85 · web-flow · commit 0828a8b0f8a1 · 2022-12-30T00:10:17.000+01:00
* Enable top level await in modules (and expressions)

* Enable test262 tests
diff --git a/src/Esprima/JavascriptParser.cs b/src/Esprima/JavascriptParser.cs
@@ -207,6 +207,7 @@ public Module ParseModule(string code, string? source = null)
         try
         {
             _context.Strict = true;
+            _context.IsAsync = true;
             _context.IsModule = true;
             _scanner._isModule = true;
 
@@ -675,7 +676,7 @@ private protected virtual Expression ParsePrimaryExpression()
         switch (_lookahead.Type)
         {
             case TokenType.Identifier:
-                if ((_context.IsModule || _context.IsAsync) && "await".Equals(_lookahead.Value))
+                if (_context.IsAsync && "await".Equals(_lookahead.Value))
                 {
                     TolerateUnexpectedToken(_lookahead);
                 }
@@ -2058,6 +2059,11 @@ private Expression ParseUnaryExpression()
         }
         else if (_context.IsAsync && MatchContextualKeyword("await"))
         {
+            if (_lookahead.End - _lookahead.Start != "await".Length)
+            {
+                TolerateUnexpectedToken(_lookahead, Messages.InvalidEscapedReservedWord);
+            }
+
             expr = ParseAwaitExpression();
         }
         else
@@ -2668,6 +2674,7 @@ public Expression ParseExpression(string code)
         Reset(code, source: null);
         try
         {
+            _context.IsAsync = true;
             return FinalizeRoot(ParseExpression());
         }
         finally
@@ -3079,7 +3086,7 @@ private Identifier ParseVariableIdentifier(VariableDeclarationKind? kind = null,
                 }
             }
         }
-        else if ((_context.IsModule || _context.IsAsync) && !allowAwaitKeyword && token.Type == TokenType.Identifier && (string?) token.Value == "await")
+        else if (_context.IsAsync && !allowAwaitKeyword && token.Type == TokenType.Identifier && (string?) token.Value == "await")
         {
             TolerateUnexpectedToken(token);
         }
diff --git a/test/Esprima.Tests.Test262/Test262Harness.settings.json b/test/Esprima.Tests.Test262/Test262Harness.settings.json
@@ -5,8 +5,7 @@
   "Namespace": "Esprima.Tests.Test262",
   "Parallel": true,
   "ExcludedFeatures": [
-    "regexp-unicode-property-escapes",
-    "top-level-await"
+    "regexp-unicode-property-escapes"
   ],
   "ExcludedFlags": [],
   "ExcludedDirectories": [],
diff --git a/test/Esprima.Tests/Fixtures.cs b/test/Esprima.Tests/Fixtures.cs
@@ -146,11 +146,14 @@ public void ExecuteTestCase(string fixture)
             metadata = FixtureMetadata.Default;
         }
 
+        if (metadata.Skip)
+        {
+            return;
+        }
+
         var parserOptions = parserOptionsFactory(false, !metadata.IgnoresRegex);
 
         var conversionOptions = metadata.CreateConversionOptions(conversionDefaultOptions);
-
-#pragma warning disable 162
         if (File.Exists(moduleFilePath))
         {
             sourceType = SourceType.Module;
@@ -182,7 +185,6 @@ public void ExecuteTestCase(string fixture)
                 if (!CompareTrees(actual, expected, metadata))
                     File.WriteAllText(failureFilePath, actual);
             }
-#pragma warning restore 162
         }
         else
         {
@@ -236,12 +238,15 @@ internal static string GetFixturesPath()
 
     private sealed class FixtureMetadata
     {
-        public static readonly FixtureMetadata Default = new FixtureMetadata(
-            testCompatibilityMode: AstToJsonTestCompatibilityMode.None,
-            includesLocation: true,
-            includesRange: true,
-            includesLocationSource: false,
-            ignoresRegex: false);
+        public static readonly FixtureMetadata Default = new()
+        {
+            TestCompatibilityMode = AstToJsonTestCompatibilityMode.None,
+            IncludesLocation = true,
+            IncludesRange = true,
+            IncludesLocationSource = false,
+            IgnoresRegex = false,
+            Skip = false,
+        };
 
         private sealed class Group
         {
@@ -276,28 +281,25 @@ public static Dictionary<string, FixtureMetadata> ReadMetadata()
 
         private static FixtureMetadata CreateFrom(HashSet<string> flags)
         {
-            return new FixtureMetadata(
-                testCompatibilityMode: flags.Contains("EsprimaOrgFixture") ? AstToJsonTestCompatibilityMode.EsprimaOrg : AstToJsonTestCompatibilityMode.None,
-                includesLocation: flags.Contains("IncludesLocation"),
-                includesRange: flags.Contains("IncludesRange"),
-                includesLocationSource: flags.Contains("IncludesLocationSource"),
-                ignoresRegex: flags.Contains("IgnoresRegex"));
+            return new FixtureMetadata
+            {
+                TestCompatibilityMode = flags.Contains("EsprimaOrgFixture") ? AstToJsonTestCompatibilityMode.EsprimaOrg : AstToJsonTestCompatibilityMode.None,
+                IncludesLocation = flags.Contains("IncludesLocation"),
+                IncludesRange = flags.Contains("IncludesRange"),
+                IncludesLocationSource = flags.Contains("IncludesLocationSource"),
+                IgnoresRegex = flags.Contains("IgnoresRegex"),
+                Skip = flags.Contains("Skip"),
+            };
         }
 
-        private FixtureMetadata(AstToJsonTestCompatibilityMode testCompatibilityMode, bool includesLocation, bool includesRange, bool includesLocationSource, bool ignoresRegex)
-        {
-            TestCompatibilityMode = testCompatibilityMode;
-            IncludesLocation = includesLocation;
-            IncludesRange = includesRange;
-            IncludesLocationSource = includesLocationSource;
-            IgnoresRegex = ignoresRegex;
-        }
+        private FixtureMetadata() { }
 
-        public AstToJsonTestCompatibilityMode TestCompatibilityMode { get; }
-        public bool IncludesLocation { get; }
-        public bool IncludesRange { get; }
-        public bool IncludesLocationSource { get; }
-        public bool IgnoresRegex { get; }
+        public AstToJsonTestCompatibilityMode TestCompatibilityMode { get; init; }
+        public bool IncludesLocation { get; init; }
+        public bool IncludesRange { get; init; }
+        public bool IncludesLocationSource { get; init; }
+        public bool IgnoresRegex { get; init; }
+        public bool Skip { get; init; }
 
         public AstToJsonOptions CreateConversionOptions(AstToJsonOptions defaultOptions) => defaultOptions with
         {
diff --git a/test/Esprima.Tests/Fixtures/fixtures-metadata.json b/test/Esprima.Tests/Fixtures/fixtures-metadata.json
@@ -619,7 +619,6 @@
       "ES6/identifier/estimated.js",
       "ES6/identifier/ethiopic_digits.js",
       "ES6/identifier/invalid_escaped_surrogate_pairs.js",
-      "ES6/identifier/invalid_expression_await.module.js",
       "ES6/identifier/invalid_function_await.module.js",
       "ES6/identifier/invalid_id_smp.js",
       "ES6/identifier/invalid_lone_surrogate.source.js",
@@ -629,7 +628,6 @@
       "ES6/identifier/math_dal_part.js",
       "ES6/identifier/math_kaf_lam.js",
       "ES6/identifier/math_zain_start.js",
-      "ES6/identifier/module_await.js",
       "ES6/identifier/valid_await.js",
       "ES6/identifier/weierstrass.js",
       "ES6/identifier/weierstrass_weierstrass.js",
@@ -1710,5 +1708,12 @@
       "expression/primary/literal/numeric/migrated_0003.js",
       "expression/primary/literal/regular-expression/migrated_0008.js"
     ]
+  },
+  {
+    "flags": [ "EsprimaOrgFixture", "Skip" ],
+    "files": [
+      "ES6/identifier/invalid_expression_await.module.js",
+      "ES6/identifier/module_await.js"
+    ]
   }
 ]
diff --git a/test/Esprima.Tests/ParserTests.cs b/test/Esprima.Tests/ParserTests.cs
@@ -616,4 +616,34 @@ class X {
 
         Assert.Equal(expected, json);
     }
+
+    [Theory]
+    [InlineData("script", true)]
+    [InlineData("module", false)]
+    [InlineData("expression", false)]
+    public void ShouldParseTopLevelAwait(string sourceType, bool shouldThrow)
+    {
+        const string code = "await import('x')";
+
+        var parser = new JavaScriptParser();
+        Func<JavaScriptParser, Node> parseAction = sourceType switch
+        {
+            "script" => parser => parser.ParseScript(code),
+            "module" => parser => parser.ParseModule(code),
+            "expression" => parser => parser.ParseExpression(code),
+            _ => throw new InvalidOperationException()
+        };
+
+        if (!shouldThrow)
+        {
+            var node = parseAction(parser);
+            var awaitExpression = node.DescendantNodesAndSelf().OfType<AwaitExpression>().FirstOrDefault();
+            Assert.NotNull(awaitExpression);
+            Assert.IsType<Import>(awaitExpression.Argument);
+        }
+        else
+        {
+            Assert.Throws<ParserException>(() => parseAction(parser));
+        }
+    }
 }

Original file line number	Diff line number	Diff line change
`@@ -207,6 +207,7 @@ public Module ParseModule(string code, string? source = null)`
`207`	`207`	`try`
`208`	`208`	`{`
`209`	`209`	`_context.Strict = true;`
	`210`	`+ _context.IsAsync = true;`
`210`	`211`	`_context.IsModule = true;`
`211`	`212`	`_scanner._isModule = true;`
`212`	`213`
`@@ -675,7 +676,7 @@ private protected virtual Expression ParsePrimaryExpression()`
`675`	`676`	`switch (_lookahead.Type)`
`676`	`677`	`{`
`677`	`678`	`case TokenType.Identifier:`
`678`		`- if ((_context.IsModule \|\| _context.IsAsync) && "await".Equals(_lookahead.Value))`
	`679`	`+ if (_context.IsAsync && "await".Equals(_lookahead.Value))`
`679`	`680`	`{`
`680`	`681`	`TolerateUnexpectedToken(_lookahead);`
`681`	`682`	`}`
`@@ -2058,6 +2059,11 @@ private Expression ParseUnaryExpression()`
`2058`	`2059`	`}`
`2059`	`2060`	`else if (_context.IsAsync && MatchContextualKeyword("await"))`
`2060`	`2061`	`{`
	`2062`	`+ if (_lookahead.End - _lookahead.Start != "await".Length)`
	`2063`	`+ {`
	`2064`	`+ TolerateUnexpectedToken(_lookahead, Messages.InvalidEscapedReservedWord);`
	`2065`	`+ }`
	`2066`	`+`
`2061`	`2067`	`expr = ParseAwaitExpression();`
`2062`	`2068`	`}`
`2063`	`2069`	`else`
`@@ -2668,6 +2674,7 @@ public Expression ParseExpression(string code)`
`2668`	`2674`	`Reset(code, source: null);`
`2669`	`2675`	`try`
`2670`	`2676`	`{`
	`2677`	`+ _context.IsAsync = true;`
`2671`	`2678`	`return FinalizeRoot(ParseExpression());`
`2672`	`2679`	`}`
`2673`	`2680`	`finally`
`@@ -3079,7 +3086,7 @@ private Identifier ParseVariableIdentifier(VariableDeclarationKind? kind = null,`
`3079`	`3086`	`}`
`3080`	`3087`	`}`
`3081`	`3088`	`}`
`3082`		`- else if ((_context.IsModule \|\| _context.IsAsync) && !allowAwaitKeyword && token.Type == TokenType.Identifier && (string?) token.Value == "await")`
	`3089`	`+ else if (_context.IsAsync && !allowAwaitKeyword && token.Type == TokenType.Identifier && (string?) token.Value == "await")`
`3083`	`3090`	`{`
`3084`	`3091`	`TolerateUnexpectedToken(token);`
`3085`	`3092`	`}`