Small perf improvement in TokenizerBackedParser.Accept and ReadWhile. (dotnet/razor#1882)

In the razor perf typing test, Accept was showing 27 ms allocating enumerators. Additionally, modified ReadWhile to only allocate if it would return a non-empty collection (and to not use the complexity introduced by using yield enumerators)\n\nCommit migrated from dotnet/razor@27a14af

Author: ToddGrun

src/Razor/Microsoft.AspNetCore.Razor.Language/src/Legacy/CSharpCodeParser.cs

@@ -893,7 +893,7 @@ private void ParseTemplate(in SyntaxListBuilder<RazorSyntaxNode> builder)
             }
         }
 
-        protected bool TryParseDirective(in SyntaxListBuilder<RazorSyntaxNode> builder, IEnumerable<SyntaxToken> whitespace, CSharpTransitionSyntax transition, string directive)
+        protected bool TryParseDirective(in SyntaxListBuilder<RazorSyntaxNode> builder, IReadOnlyList<SyntaxToken> whitespace, CSharpTransitionSyntax transition, string directive)
         {
             if (_directiveParserMap.TryGetValue(directive, out var handler))
             {
@@ -1679,7 +1679,7 @@ private void ParseDirectiveBlock(in SyntaxListBuilder<RazorSyntaxNode> builder,
             }
         }
 
-        private bool TryParseKeyword(in SyntaxListBuilder<RazorSyntaxNode> builder, IEnumerable<SyntaxToken> whitespace, CSharpTransitionSyntax transition)
+        private bool TryParseKeyword(in SyntaxListBuilder<RazorSyntaxNode> builder, IReadOnlyList<SyntaxToken> whitespace, CSharpTransitionSyntax transition)
         {
             var result = CSharpTokenizer.GetTokenKeyword(CurrentToken);
             Debug.Assert(CurrentToken.Kind == SyntaxKind.Keyword && result.HasValue);
@@ -2387,7 +2387,7 @@ protected void CompleteBlock(bool insertMarkerIfNecessary, bool captureWhitespac
             }
         }
 
-        private IEnumerable<SyntaxToken> SkipToNextImportantToken(in SyntaxListBuilder<RazorSyntaxNode> builder)
+        private IReadOnlyList<SyntaxToken> SkipToNextImportantToken(in SyntaxListBuilder<RazorSyntaxNode> builder)
         {
             while (!EndOfFile)
             {
@@ -2406,7 +2406,7 @@ private IEnumerable<SyntaxToken> SkipToNextImportantToken(in SyntaxListBuilder<R
                     return whitespace;
                 }
             }
-            return Enumerable.Empty<SyntaxToken>();
+            return Array.Empty<SyntaxToken>();
         }
 
         private void DefaultSpanContextConfig(SpanContextBuilder spanContext)

src/Razor/Microsoft.AspNetCore.Razor.Language/src/Legacy/HtmlMarkupParser.cs

@@ -1085,9 +1085,9 @@ private void ParseAttribute(in SyntaxListBuilder<RazorSyntaxNode> builder)
             }
         }
 
-        private bool TryParseAttributeName(out IEnumerable<SyntaxToken> nameTokens)
+        private bool TryParseAttributeName(out IReadOnlyList<SyntaxToken> nameTokens)
         {
-            nameTokens = Enumerable.Empty<SyntaxToken>();
+            nameTokens = Array.Empty<SyntaxToken>();
             //
             // We are currently here <input |name="..." />
             // If we encounter a transition (@) here, it can be parsed as CSharp or Markup depending on the feature flag.

src/Razor/Microsoft.AspNetCore.Razor.Language/src/Legacy/TokenizerBackedParser.cs

@@ -266,14 +266,22 @@ protected bool EnsureCurrent()
             return true;
         }
 
-        protected internal IEnumerable<SyntaxToken> ReadWhile(params SyntaxKind[] types)
-        {
-            return ReadWhile(token => types.Any(expected => expected == token.Kind));
-        }
+        protected internal IReadOnlyList<SyntaxToken> ReadWhile(Func<SyntaxToken, bool> condition)
+        {
+            if (!EnsureCurrent() || !condition(CurrentToken))
+            {
+                return Array.Empty<SyntaxToken>();
+            }
+
+            var result = new List<SyntaxToken>();
+            do
+            {
+                result.Add(CurrentToken);
+                NextToken();
+            }
+            while (EnsureCurrent() && condition(CurrentToken));
 
-        protected internal IEnumerable<SyntaxToken> ReadWhile(Func<SyntaxToken, bool> condition)
-        {
-            return ReadWhileLazy(condition).ToList();
+            return result;
         }
 
         protected bool AtIdentifier(bool allowKeywords)
@@ -283,17 +291,6 @@ protected bool AtIdentifier(bool allowKeywords)
                     (allowKeywords && Language.IsKeyword(CurrentToken)));
         }
 
-        // Don't open this to sub classes because it's lazy but it looks eager.
-        // You have to advance the Enumerable to read the next characters.
-        internal IEnumerable<SyntaxToken> ReadWhileLazy(Func<SyntaxToken, bool> condition)
-        {
-            while (EnsureCurrent() && condition(CurrentToken))
-            {
-                yield return CurrentToken;
-                NextToken();
-            }
-        }
-
         protected RazorCommentBlockSyntax ParseRazorComment()
         {
             if (!Language.KnowsTokenType(KnownTokenType.CommentStart) ||
@@ -430,13 +427,14 @@ protected internal void AcceptUntil(params SyntaxKind[] types)
 
         protected internal void AcceptWhile(Func<SyntaxToken, bool> condition)
         {
-            Accept(ReadWhileLazy(condition));
+            Accept(ReadWhile(condition));
         }
 
-        protected internal void Accept(IEnumerable<SyntaxToken> tokens)
+        protected internal void Accept(IReadOnlyList<SyntaxToken> tokens)
         {
-            foreach (var token in tokens)
+            for(int i = 0; i < tokens.Count; i++)
             {
+                var token = tokens[i];
                 Accept(token);
             }
         }