DEV: Use markdown-it for better img markdown processing

Author: keegangeorge

assets/javascripts/discourse/components/modal/diff-modal.gjs

@@ -145,6 +145,7 @@ export default class ModalDiffModal extends Component {
                 "streamable-content"
                 (if this.isStreaming "streaming")
                 (if @model.showResultAsDiff "inline-diff")
+                (if this.diffStreamer.isThinking "thinking")
               }}
             >
               {{~#if @model.showResultAsDiff~}}

assets/javascripts/discourse/lib/diff-streamer.gjs

@@ -1,7 +1,7 @@
 import { tracked } from "@glimmer/tracking";
 import { later } from "@ember/runloop";
 import loadJSDiff from "discourse/lib/load-js-diff";
-import { IMAGE_MARKDOWN_REGEX } from "discourse/lib/uploads";
+import { parseAsync } from "discourse/lib/text";
 
 const DEFAULT_CHAR_TYPING_DELAY = 30;
 
@@ -12,6 +12,7 @@ export default class DiffStreamer {
   @tracked diff = "";
   @tracked suggestion = "";
   @tracked isDone = false;
+  @tracked isThinking = false;
 
   typingTimer = null;
   currentWordIndex = 0;
@@ -37,6 +38,7 @@ export default class DiffStreamer {
     this.isDone = !!result?.done;
 
     if (newText.length < this.lastResultText.length) {
+      this.isThinking = false;
       // reset if text got shorter (e.g., reset or new input)
       this.words = [];
       this.suggestion = "";
@@ -45,11 +47,17 @@ export default class DiffStreamer {
     }
 
     const diffText = newText.slice(this.lastResultText.length);
+
     if (!diffText.trim()) {
       this.lastResultText = newText;
       return;
     }
 
+    if (await this.#isIncompleteMarkdown(diffText)) {
+      this.isThinking = true;
+      return;
+    }
+
     const newWords = this.#tokenizeMarkdownAware(diffText);
 
     if (newWords.length > 0) {
@@ -63,32 +71,6 @@ export default class DiffStreamer {
     this.lastResultText = newText;
   }
 
-  #tokenizeMarkdownAware(text) {
-    const tokens = [];
-    let lastIndex = 0;
-
-    let match;
-    while ((match = IMAGE_MARKDOWN_REGEX.exec(text)) !== null) {
-      const matchStart = match.index;
-
-      if (lastIndex < matchStart) {
-        const preceding = text.slice(lastIndex, matchStart);
-        tokens.push(...(preceding.match(/\S+\s*|\s+/g) || []));
-      }
-
-      tokens.push(match[0]);
-
-      lastIndex = IMAGE_MARKDOWN_REGEX.lastIndex;
-    }
-
-    if (lastIndex < text.length) {
-      const trailing = text.slice(lastIndex);
-      tokens.push(...(trailing.match(/\S+\s*|\s+/g) || []));
-    }
-
-    return tokens;
-  }
-
   reset() {
     this.diff = "";
     this.suggestion = "";
@@ -103,16 +85,34 @@ export default class DiffStreamer {
     }
   }
 
+  async #isIncompleteMarkdown(text) {
+    const tokens = await parseAsync(text);
+
+    const hasImage = tokens.some((t) => t.type === "image");
+    const hasLink = tokens.some((t) => t.type === "link_open");
+
+    if (hasImage || hasLink) {
+      return false;
+    }
+
+    const maybeUnfinishedImage =
+      /!\[[^\]]*$/.test(text) || /!\[[^\]]*]\(upload:\/\/[^\s)]+$/.test(text);
+
+    const maybeUnfinishedLink =
+      /\[[^\]]*$/.test(text) || /\[[^\]]*]\([^\s)]+$/.test(text);
+
+    return maybeUnfinishedImage || maybeUnfinishedLink;
+  }
+
   async #streamNextChar() {
     if (this.currentWordIndex < this.words.length) {
       const currentToken = this.words[this.currentWordIndex];
 
-      const isMarkdownToken =
-        currentToken.startsWith("![") || currentToken.startsWith("[");
-
-      if (isMarkdownToken) {
-        this.suggestion += currentToken;
+      const nextChar = currentToken.charAt(this.currentCharIndex);
+      this.suggestion += nextChar;
+      this.currentCharIndex++;
 
+      if (this.currentCharIndex >= currentToken.length) {
         this.currentWordIndex++;
         this.currentCharIndex = 0;
 
@@ -126,31 +126,9 @@ export default class DiffStreamer {
         if (this.currentWordIndex === 1) {
           this.diff = this.diff.replace(/^\s+/, "");
         }
-
-        this.typingTimer = later(this, this.#streamNextChar, this.typingDelay);
-      } else {
-        const nextChar = currentToken.charAt(this.currentCharIndex);
-        this.suggestion += nextChar;
-        this.currentCharIndex++;
-
-        if (this.currentCharIndex >= currentToken.length) {
-          this.currentWordIndex++;
-          this.currentCharIndex = 0;
-
-          const originalDiff = this.jsDiff.diffWordsWithSpace(
-            this.selectedText,
-            this.suggestion
-          );
-
-          this.diff = this.#formatDiffWithTags(originalDiff);
-
-          if (this.currentWordIndex === 1) {
-            this.diff = this.diff.replace(/^\s+/, "");
-          }
-        }
-
-        this.typingTimer = later(this, this.#streamNextChar, this.typingDelay);
       }
+
+      this.typingTimer = later(this, this.#streamNextChar, this.typingDelay);
     } else {
       if (!this.suggestion || !this.selectedText || !this.jsDiff) {
         return;
@@ -167,6 +145,33 @@ export default class DiffStreamer {
     }
   }
 
+  #tokenizeMarkdownAware(text) {
+    const tokens = [];
+    let lastIndex = 0;
+    const regex = /!\[[^\]]*]\(upload:\/\/[^\s)]+\)/g;
+
+    let match;
+    while ((match = regex.exec(text)) !== null) {
+      const matchStart = match.index;
+
+      if (lastIndex < matchStart) {
+        const before = text.slice(lastIndex, matchStart);
+        tokens.push(...(before.match(/\S+\s*|\s+/g) || []));
+      }
+
+      tokens.push(match[0]);
+
+      lastIndex = regex.lastIndex;
+    }
+
+    if (lastIndex < text.length) {
+      const rest = text.slice(lastIndex);
+      tokens.push(...(rest.match(/\S+\s*|\s+/g) || []));
+    }
+
+    return tokens;
+  }
+
   #wrapChunk(text, type) {
     if (type === "added") {
       return `<ins>${text}</ins>`;

assets/stylesheets/common/streaming.scss

@@ -83,19 +83,34 @@ article.streaming .cooked {
 @keyframes mark-blink {
   0%,
   100% {
-    border-color: var(--highlight-high);
+    border-color: transparent;
   }
 
   50% {
-    border-color: transparent;
+    border-color: var(--highlight-high);
   }
 }
 
+@keyframes fade-in-highlight {
+  from {
+    opacity: 0.5;
+  }
+
+  to {
+    opacity: 1;
+  }
+}
+
+mark.highlight {
+  background-color: var(--highlight-high);
+  animation: fade-in-highlight 0.5s ease-in-out forwards;
+}
+
 .composer-ai-helper-modal__suggestion.thinking mark.highlight {
   animation: mark-blink 1s step-start 0s infinite;
+  animation-name: mark-blink;
 }
 
-// TODO: cleanup and move around to proper files
 .composer-ai-helper-modal__loading {
   white-space: pre-wrap;
 }
@@ -113,17 +128,3 @@ article.streaming .cooked {
     display: inline;
   }
 }
-
-mark.highlight {
-  background-color: var(--highlight-high);
-  animation: fadeInHighlight 0.5s ease-in-out forwards;
-}
-
-@keyframes fadeInHighlight {
-  from {
-    opacity: 0.5;
-  }
-  to {
-    opacity: 1;
-  }
-}