Hugelung/rich mcp response (RooCodeInc#1941)

* showing images after mcp responses

* images now open in a webview tab

* Open Graph link metadata display for MCP responses

* almost totally working rich mcp response display with images and embeds

* closer

* header for response display

* updated styling of mcp responses

* default to plain text if rich response is loading

* formatting fix

* added changeset output

* remove some old code

* add the dashed border back

* avoid XSS attacks by sanitizing the preview image urls and embeds

* remove incorrect vendor prefix css

* delete old version of open image implementation

* undo some comment removals and cleanups to make PR easier to read

---------

Co-authored-by: Andrei Edell <[email protected]>

Author: 2 people

.changeset/old-dancers-smell.md

@@ -0,0 +1,5 @@
+---
+"claude-dev": minor
+---
+
+Add rich MCP responses with images and link previews

package-lock.json

@@ -264,6 +264,7 @@
 		"isbinaryfile": "^5.0.2",
 		"mammoth": "^1.8.0",
 		"monaco-vscode-textmate-theme-converter": "^0.1.7",
+		"open-graph-scraper": "^6.9.0",
 		"openai": "^4.83.0",
 		"os-name": "^6.0.0",
 		"p-wait-for": "^5.0.2",

package.json

@@ -10,6 +10,7 @@ import * as vscode from "vscode"
 import { buildApiHandler } from "../../api"
 import { downloadTask } from "../../integrations/misc/export-markdown"
 import { openFile, openImage } from "../../integrations/misc/open-file"
+import { fetchOpenGraphData, isImageUrl } from "../../integrations/misc/link-preview"
 import { selectImages } from "../../integrations/misc/process-images"
 import { getTheme } from "../../integrations/theme/getTheme"
 import WorkspaceTracker from "../../integrations/workspace/WorkspaceTracker"
@@ -663,6 +664,17 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 					case "openImage":
 						openImage(message.text!)
 						break
+					case "openInBrowser":
+						if (message.url) {
+							vscode.env.openExternal(vscode.Uri.parse(message.url))
+						}
+						break
+					case "fetchOpenGraphData":
+						this.fetchOpenGraphData(message.text!)
+						break
+					case "checkIsImageUrl":
+						this.checkIsImageUrl(message.text!)
+						break
 					case "openFile":
 						openFile(message.text!)
 						break
@@ -1955,6 +1967,53 @@ Here is the project's README to help you get started:\n\n${mcpDetails.readmeCont
 		return await this.context.secrets.get(key)
 	}
 
+	// Open Graph Data
+
+	async fetchOpenGraphData(url: string) {
+		try {
+			// Use the fetchOpenGraphData function from link-preview.ts
+			const ogData = await fetchOpenGraphData(url)
+
+			// Send the data back to the webview
+			await this.postMessageToWebview({
+				type: "openGraphData",
+				openGraphData: ogData,
+				url: url,
+			})
+		} catch (error) {
+			console.error(`Error fetching Open Graph data for ${url}:`, error)
+			// Send an error response
+			await this.postMessageToWebview({
+				type: "openGraphData",
+				error: `Failed to fetch Open Graph data: ${error}`,
+				url: url,
+			})
+		}
+	}
+
+	// Check if a URL is an image
+	async checkIsImageUrl(url: string) {
+		try {
+			// Check if the URL is an image
+			const isImage = await isImageUrl(url)
+
+			// Send the result back to the webview
+			await this.postMessageToWebview({
+				type: "isImageUrlResult",
+				isImage,
+				url,
+			})
+		} catch (error) {
+			console.error(`Error checking if URL is an image: ${url}`, error)
+			// Send an error response
+			await this.postMessageToWebview({
+				type: "isImageUrlResult",
+				isImage: false,
+				url,
+			})
+		}
+	}
+
 	// dev
 
 	async resetState() {

src/core/webview/ClineProvider.ts

@@ -0,0 +1,107 @@
+import axios from "axios"
+import ogs from "open-graph-scraper"
+
+export interface OpenGraphData {
+	title?: string
+	description?: string
+	image?: string
+	url?: string
+	siteName?: string
+	type?: string
+}
+
+/**
+ * Fetches Open Graph metadata from a URL
+ * @param url The URL to fetch metadata from
+ * @returns Promise resolving to OpenGraphData
+ */
+export async function fetchOpenGraphData(url: string): Promise<OpenGraphData> {
+	try {
+		const options = {
+			url: url,
+			timeout: 5000,
+			headers: {
+				"user-agent": "Mozilla/5.0 (compatible; VSCodeExtension/1.0; +https://cline.bot)",
+			},
+			onlyGetOpenGraphInfo: false, // Get all metadata, not just Open Graph
+			fetchOptions: {
+				redirect: "follow", // Follow redirects
+			} as any,
+		}
+
+		const { result } = await ogs(options)
+
+		// Use type assertion to avoid TypeScript errors
+		const data = result as any
+
+		// Handle image URLs
+		let imageUrl = data.ogImage?.[0]?.url || data.twitterImage?.[0]?.url
+
+		// If the image URL is relative, make it absolute
+		if (imageUrl && (imageUrl.startsWith("/") || imageUrl.startsWith("./"))) {
+			try {
+				// Extract the base URL and make the relative URL absolute
+				const urlObj = new URL(url)
+				const baseUrl = `${urlObj.protocol}//${urlObj.hostname}`
+				imageUrl = new URL(imageUrl, baseUrl).href
+			} catch (error) {
+				console.error(`Error converting relative URL to absolute: ${imageUrl}`, error)
+			}
+		}
+
+		return {
+			title: data.ogTitle || data.twitterTitle || data.dcTitle || data.title || new URL(url).hostname,
+			description:
+				data.ogDescription ||
+				data.twitterDescription ||
+				data.dcDescription ||
+				data.description ||
+				"No description available",
+			image: imageUrl,
+			url: data.ogUrl || url,
+			siteName: data.ogSiteName || new URL(url).hostname,
+			type: data.ogType,
+		}
+	} catch (error) {
+		console.error(`Error fetching Open Graph data for ${url}:`, error)
+		// Return basic information based on the URL
+		try {
+			const urlObj = new URL(url)
+			return {
+				title: urlObj.hostname,
+				description: url,
+				url: url,
+				siteName: urlObj.hostname,
+			}
+		} catch {
+			return {
+				title: url,
+				description: url,
+				url: url,
+			}
+		}
+	}
+}
+
+/**
+ * Checks if a URL is an image by making a HEAD request and checking the content type
+ * @param url The URL to check
+ * @returns Promise resolving to boolean indicating if the URL is an image
+ */
+export async function isImageUrl(url: string): Promise<boolean> {
+	try {
+		const response = await axios.head(url, {
+			headers: {
+				"User-Agent": "Mozilla/5.0 (compatible; VSCodeExtension/1.0; +https://cline.bot)",
+			},
+			timeout: 3000,
+		})
+
+		const contentType = response.headers["content-type"]
+		return contentType && contentType.startsWith("image/")
+	} catch (error) {
+		console.error(`Error checking if URL is an image: ${url}`, error)
+		// If we can't determine, fall back to checking the file extension
+		return /\.(jpg|jpeg|png|gif|webp|svg)$/i.test(url)
+	}
+}

src/integrations/misc/link-preview.ts

@@ -31,6 +31,8 @@ export interface ExtensionMessage {
 		| "mcpMarketplaceCatalog"
 		| "mcpDownloadDetails"
 		| "commitSearchResults"
+		| "openGraphData"
+		| "isImageUrlResult"
 	text?: string
 	action?:
 		| "chatButtonClicked"
@@ -55,6 +57,16 @@ export interface ExtensionMessage {
 	error?: string
 	mcpDownloadDetails?: McpDownloadResponse
 	commits?: GitCommit[]
+	openGraphData?: {
+		title?: string
+		description?: string
+		image?: string
+		url?: string
+		siteName?: string
+		type?: string
+	}
+	url?: string
+	isImage?: boolean
 }
 
 export type Platform = "aix" | "darwin" | "freebsd" | "linux" | "openbsd" | "sunos" | "win32" | "unknown"

src/shared/ExtensionMessage.ts

@@ -22,6 +22,7 @@ export interface WebviewMessage {
 		| "requestOllamaModels"
 		| "requestLmStudioModels"
 		| "openImage"
+		| "openInBrowser"
 		| "openFile"
 		| "openMention"
 		| "cancelTask"
@@ -52,6 +53,9 @@ export interface WebviewMessage {
 		| "fetchLatestMcpServersFromHub"
 		| "telemetrySetting"
 		| "openSettings"
+		| "updateMcpTimeout"
+		| "fetchOpenGraphData"
+		| "checkIsImageUrl"
 	// | "relaunchChromeDebugMode"
 	text?: string
 	disabled?: boolean
@@ -70,6 +74,9 @@ export interface WebviewMessage {
 	serverName?: string
 	toolName?: string
 	autoApprove?: boolean
+
+	// For openInBrowser
+	url?: string
 }
 
 export type ClineAskResponse = "yesButtonClicked" | "noButtonClicked" | "messageResponse"

src/shared/WebviewMessage.ts

@@ -4,8 +4,10 @@
 	"private": true,
 	"dependencies": {
 		"@floating-ui/react": "^0.27.4",
+		"@types/dompurify": "^3.0.5",
 		"@vscode/webview-ui-toolkit": "^1.4.0",
 		"debounce": "^2.1.1",
+		"dompurify": "^3.2.4",
 		"fast-deep-equal": "^3.1.3",
 		"fuse.js": "^7.0.0",
 		"fzf": "^0.5.2",