Fix GenerateRender to not throw 'IndexOutOfRangeException' and 'NullReferenceException' (#1049)

Author: daxian-dbw

PSReadLine/Render.cs

@@ -226,26 +226,32 @@ void MaybeEmphasize(int i, string currColor)
                     var token = state.Tokens[state.Index];
                     while (i == token.Extent.EndOffset)
                     {
-                        if (token == state.Tokens[state.Tokens.Length - 1])
+                        if (state.Index == state.Tokens.Length - 1)
                         {
                             tokenStack.Pop();
                             if (tokenStack.Count == 0)
                             {
                                 afterLastToken = true;
                                 token = null;
                                 color = defaultColor;
+                                break;
                             }
                             else
                             {
                                 state = tokenStack.Peek();
+
+                                // It's possible that a 'StringExpandableToken' is the last available token, for example:
+                                //   'begin $a\abc def', 'process $a\abc | blah' and 'end $a\abc; hello'
+                                // due to the special handling of the keywords 'begin', 'process' and 'end', all the above 3 script inputs
+                                // generate only 2 tokens by the parser -- A KeywordToken, and a StringExpandableToken '$a\abc'. Text after
+                                // '$a\abc' is not tokenized at all.
+                                // We repeat the test to see if we fall into this case ('token' is the final one in the stack).
+                                continue;
                             }
                         }
 
-                        if (!afterLastToken)
-                        {
-                            color = state.Color;
-                            token = state.Tokens[++state.Index];
-                        }
+                        color = state.Color;
+                        token = state.Tokens[++state.Index];
                     }
 
                     if (!afterLastToken && i == token.Extent.StartOffset)

test/RenderTest.cs

@@ -94,6 +94,30 @@ public void Render()
                 InputAcceptedNow
                 ));
 
+            // test that rendering doesn't cause an exception in a potential missing "EOS" token case.
+            // this case could be a moving target, if the PowerShell parser is changed such as to eliminate the case.
+            Test("", Keys(
+                "process $abc\\name | def",
+                CheckThat(() =>
+                    AssertScreenIs(1,
+                        TokenClassification.Keyword, "process",
+                        TokenClassification.None, " ",
+                        TokenClassification.Variable, "$abc",
+                        TokenClassification.None, "\\name | def")),
+                _.Ctrl_c,
+                InputAcceptedNow
+                ));
+
+            Test("", Keys(
+                "process out put",
+                CheckThat(() =>
+                    AssertScreenIs(1,
+                        TokenClassification.Keyword, "process",
+                        TokenClassification.None, " out put")),
+                _.Ctrl_c,
+                InputAcceptedNow
+                ));
+
             Test("", Keys(
                 "\"$([int];\"_$(1+2)\")\"",
                 CheckThat(() =>