feat: add integrated web preview with element selection (fixes #5971)

- Add WebPreviewProvider to manage preview panel in sidebar
- Create React-based UI with URL navigation and viewport controls
- Implement element selection with context extraction (HTML, CSS, XPath)
- Add web_preview tool for AI integration
- Register commands and update package.json
- Update type definitions and tool configurations

This implementation provides:
- Web preview panel in VSCode sidebar
- Click-to-select elements with visual highlighting
- Element context extraction for AI assistance
- Responsive viewport controls
- Integration with Roo Code AI through web_preview tool

Author: roomote

packages/types/src/tool.ts

@@ -34,6 +34,7 @@ export const toolNames = [
 	"fetch_instructions",
 	"codebase_search",
 	"update_todo_list",
+	"web_preview",
 ] as const
 
 export const toolNamesSchema = z.enum(toolNames)

packages/types/src/vscode.ts

@@ -53,6 +53,8 @@ export const commandIds = [
 	"focusInput",
 	"acceptInput",
 	"focusPanel",
+	"openWebPreview",
+	"getSelectedElement",
 ] as const
 
 export type CommandId = (typeof commandIds)[number]

src/activate/registerCommands.ts

@@ -7,6 +7,7 @@ import { TelemetryService } from "@roo-code/telemetry"
 import { Package } from "../shared/package"
 import { getCommand } from "../utils/commands"
 import { ClineProvider } from "../core/webview/ClineProvider"
+import { WebPreviewProvider } from "../core/webview/WebPreviewProvider"
 import { ContextProxy } from "../core/config/ContextProxy"
 import { focusPanel } from "../utils/focusPanel"
 
@@ -32,6 +33,7 @@ export function getVisibleProviderOrLog(outputChannel: vscode.OutputChannel): Cl
 // Store panel references in both modes
 let sidebarPanel: vscode.WebviewView | undefined = undefined
 let tabPanel: vscode.WebviewPanel | undefined = undefined
+let webPreviewProvider: WebPreviewProvider | undefined = undefined
 
 /**
  * Get the currently active panel
@@ -57,6 +59,10 @@ export function setPanel(
 	}
 }
 
+export function setWebPreviewProvider(provider: WebPreviewProvider) {
+	webPreviewProvider = provider
+}
+
 export type RegisterCommandOptions = {
 	context: vscode.ExtensionContext
 	outputChannel: vscode.OutputChannel
@@ -218,6 +224,27 @@ const getCommandsMap = ({ context, outputChannel, provider }: RegisterCommandOpt
 
 		visibleProvider.postMessageToWebview({ type: "acceptInput" })
 	},
+	openWebPreview: async (url?: string) => {
+		if (!webPreviewProvider) {
+			outputChannel.appendLine("Web Preview Provider not initialized")
+			return
+		}
+
+		if (url) {
+			await webPreviewProvider.openUrl(url)
+		} else {
+			// Show the web preview panel without navigating
+			await vscode.commands.executeCommand("roo-code.webPreview.focus")
+		}
+	},
+	getSelectedElement: async () => {
+		if (!webPreviewProvider) {
+			outputChannel.appendLine("Web Preview Provider not initialized")
+			return null
+		}
+
+		return webPreviewProvider.getSelectedElementContext()
+	},
 })
 
 export const openClineInNewTab = async ({ context, outputChannel }: Omit<RegisterCommandOptions, "provider">) => {

src/core/assistant-message/presentAssistantMessage.ts

@@ -24,6 +24,7 @@ import { askFollowupQuestionTool } from "../tools/askFollowupQuestionTool"
 import { switchModeTool } from "../tools/switchModeTool"
 import { attemptCompletionTool } from "../tools/attemptCompletionTool"
 import { newTaskTool } from "../tools/newTaskTool"
+import { webPreviewTool } from "../tools/webPreviewTool"
 
 import { checkpointSave } from "../checkpoints"
 import { updateTodoListTool } from "../tools/updateTodoListTool"
@@ -214,6 +215,8 @@ export async function presentAssistantMessage(cline: Task) {
 						const modeName = getModeBySlug(mode, customModes)?.name ?? mode
 						return `[${block.name} in ${modeName} mode: '${message}']`
 					}
+					case "web_preview":
+						return `[${block.name} for '${block.params.action}'${block.params.url ? ` - ${block.params.url}` : ""}]`
 				}
 			}
 
@@ -522,6 +525,9 @@ export async function presentAssistantMessage(cline: Task) {
 						askFinishSubTaskApproval,
 					)
 					break
+				case "web_preview":
+					await webPreviewTool(cline, block, askApproval, handleError, pushToolResult)
+					break
 			}
 
 			break

src/core/tools/webPreviewTool.ts

@@ -0,0 +1,91 @@
+import * as vscode from "vscode"
+import { Task } from "../task/Task"
+import { ClineSayTool } from "../../shared/ExtensionMessage"
+import { formatResponse } from "../prompts/responses"
+import { ToolUse, AskApproval, HandleError, PushToolResult } from "../../shared/tools"
+
+export async function webPreviewTool(
+	cline: Task,
+	block: ToolUse,
+	askApproval: AskApproval,
+	handleError: HandleError,
+	pushToolResult: PushToolResult,
+) {
+	const action = block.params.action
+	const url = block.params.url
+
+	if (!action) {
+		cline.consecutiveMistakeCount++
+		const errorMsg = await cline.sayAndCreateMissingParamError("web_preview", "action")
+		pushToolResult(formatResponse.toolError(errorMsg))
+		return
+	}
+
+	if (action === "open" && !url) {
+		cline.consecutiveMistakeCount++
+		const errorMsg = await cline.sayAndCreateMissingParamError("web_preview", "url")
+		pushToolResult(formatResponse.toolError(errorMsg))
+		return
+	}
+
+	try {
+		// Handle partial message
+		if (block.partial) {
+			const partialMessage = JSON.stringify({
+				tool: "web_preview",
+				action,
+				url,
+			} satisfies ClineSayTool)
+			await cline.ask("tool", partialMessage, block.partial).catch(() => {})
+			return
+		}
+
+		// Ask for approval
+		const completeMessage = JSON.stringify({
+			tool: "web_preview",
+			action,
+			url,
+		} satisfies ClineSayTool)
+
+		const { response, text, images } = await cline.ask("tool", completeMessage, false)
+
+		if (response !== "yesButtonClicked") {
+			if (text) {
+				await cline.say("user_feedback", text, images)
+			}
+			cline.didRejectTool = true
+			pushToolResult(formatResponse.toolDenied())
+			return
+		}
+
+		if (text) {
+			await cline.say("user_feedback", text, images)
+		}
+
+		// Execute the web preview action
+		if (action === "open") {
+			// Open the web preview panel with the URL
+			await vscode.commands.executeCommand("roo-code.openWebPreview", url)
+			pushToolResult(formatResponse.toolResult(`Opened web preview for: ${url}`))
+		} else if (action === "select") {
+			// Get the current selected element context from the preview
+			const selectedContext = await vscode.commands.executeCommand("roo-code.getSelectedElement")
+			if (selectedContext) {
+				pushToolResult(
+					formatResponse.toolResult(`Selected element context:\n${JSON.stringify(selectedContext, null, 2)}`),
+				)
+			} else {
+				pushToolResult(
+					formatResponse.toolResult(
+						"No element is currently selected. Click on an element in the preview to select it.",
+					),
+				)
+			}
+		} else {
+			pushToolResult(formatResponse.toolError(`Unknown action: ${action}. Valid actions are: open, select`))
+		}
+	} catch (error) {
+		await handleError("web preview operation", error instanceof Error ? error : new Error(String(error)))
+		pushToolResult(formatResponse.toolError(error instanceof Error ? error.message : String(error)))
+	}
+}