Fix some markdown streaming issues (microsoft#252816)

* Fix incomplete markdown rendering issue with trailing spaces

* Fix issue with trailing dash

Author: roblourens

src/vs/base/browser/markdownRenderer.ts

@@ -827,9 +827,9 @@ function completeListItemPattern(list: marked.Tokens.List): marked.Tokens.List |
 	return newList;
 }
 
-function completeHeading(token: marked.Tokens.Heading): marked.TokensList | void {
+function completeHeading(token: marked.Tokens.Heading, fullRawText: string): marked.TokensList | void {
 	if (token.raw.match(/-\s*$/)) {
-		return marked.lexer(token.raw + '  ');
+		return marked.lexer(fullRawText + '  ');
 	}
 }
 
@@ -887,7 +887,7 @@ function fillInIncompleteTokensOnce(tokens: marked.TokensList): marked.TokensLis
 	}
 
 	if (lastToken?.type === 'heading') {
-		const completeTokens = completeHeading(lastToken as marked.Tokens.Heading);
+		const completeTokens = completeHeading(lastToken as marked.Tokens.Heading, mergeRawTokenText(tokens));
 		if (completeTokens) {
 			return completeTokens;
 		}
@@ -910,15 +910,15 @@ function completeUnderscore(tokens: marked.Token): marked.Token {
 }
 
 function completeLinkTarget(tokens: marked.Token): marked.Token {
-	return completeWithString(tokens, ')');
+	return completeWithString(tokens, ')', false);
 }
 
 function completeLinkTargetArg(tokens: marked.Token): marked.Token {
-	return completeWithString(tokens, '")');
+	return completeWithString(tokens, '")', false);
 }
 
 function completeLinkText(tokens: marked.Token): marked.Token {
-	return completeWithString(tokens, '](https://microsoft.com)');
+	return completeWithString(tokens, '](https://microsoft.com)', false);
 }
 
 function completeDoublestar(tokens: marked.Token): marked.Token {
@@ -929,12 +929,13 @@ function completeDoubleUnderscore(tokens: marked.Token): marked.Token {
 	return completeWithString(tokens, '__');
 }
 
-function completeWithString(tokens: marked.Token[] | marked.Token, closingString: string): marked.Token {
+function completeWithString(tokens: marked.Token[] | marked.Token, closingString: string, shouldTrim = true): marked.Token {
 	const mergedRawText = mergeRawTokenText(Array.isArray(tokens) ? tokens : [tokens]);
 
 	// If it was completed correctly, this should be a single token.
 	// Expecting either a Paragraph or a List
-	return marked.lexer(mergedRawText + closingString)[0] as marked.Token;
+	const trimmedRawText = shouldTrim ? mergedRawText.trimEnd() : mergedRawText;
+	return marked.lexer(trimmedRawText + closingString)[0] as marked.Token;
 }
 
 function completeTable(tokens: marked.Token[]): marked.Token[] | undefined {

src/vs/base/test/browser/markdownRenderer.test.ts

@@ -511,6 +511,15 @@ suite('MarkdownRenderer', () => {
 				assert.deepStrictEqual(newTokens, completeTokens);
 			});
 
+			test(`${name} with trailing space`, () => {
+				const incomplete = `some text and ${delimiter}some code `;
+				const tokens = marked.marked.lexer(incomplete);
+				const newTokens = fillInIncompleteTokens(tokens);
+
+				const completeTokens = marked.marked.lexer(incomplete.trimEnd() + delimiter);
+				assert.deepStrictEqual(newTokens, completeTokens);
+			});
+
 			test(`single loose "${delimiter}"`, () => {
 				const text = `some text and ${delimiter}by itself\nmore text here`;
 				const tokens = marked.marked.lexer(text);
@@ -714,14 +723,23 @@ suite('MarkdownRenderer', () => {
 				assert.deepStrictEqual(newTokens, completeTokens);
 			});
 
-			test('text with start of list', () => {
+			test('text with start of list is not a heading', () => {
 				const incomplete = `hello\n- `;
 				const tokens = marked.marked.lexer(incomplete);
 				const newTokens = fillInIncompleteTokens(tokens);
 
 				const completeTokens = marked.marked.lexer(incomplete + '  ');
 				assert.deepStrictEqual(newTokens, completeTokens);
 			});
+
+			test('even more text with start of list is not a heading', () => {
+				const incomplete = `# hello\n\ntext\n-`;
+				const tokens = marked.marked.lexer(incomplete);
+				const newTokens = fillInIncompleteTokens(tokens);
+
+				const completeTokens = marked.marked.lexer(incomplete + '  ');
+				assert.deepStrictEqual(newTokens, completeTokens);
+			});
 		});
 
 		suite('codespan', () => {
@@ -779,6 +797,16 @@ suite('MarkdownRenderer', () => {
 				const completeTokens = marked.marked.lexer(text + '**');
 				assert.deepStrictEqual(newTokens, completeTokens);
 			});
+
+			// TODO trim these patterns from end
+			test.skip(`ending in doublestar`, () => {
+				const incomplete = `some text and **`;
+				const tokens = marked.marked.lexer(incomplete);
+				const newTokens = fillInIncompleteTokens(tokens);
+
+				const completeTokens = marked.marked.lexer(incomplete.trimEnd() + '**');
+				assert.deepStrictEqual(newTokens, completeTokens);
+			});
 		});
 
 		suite('underscore', () => {