Add browser settings to change headless mode and size

Author: saoudrizwan

package-lock.json

@@ -56,6 +56,7 @@ import { fixModelHtmlEscaping } from "../utils/string"
 import { OpenAiHandler } from "../api/providers/openai"
 import CheckpointTracker from "../integrations/checkpoints/CheckpointTracker"
 import getFolderSize from "get-folder-size"
+import { BrowserSettings } from "../shared/BrowserSettings"
 
 const cwd = vscode.workspace.workspaceFolders?.map((folder) => folder.uri.fsPath).at(0) ?? path.join(os.homedir(), "Desktop") // may or may not exist but fs checking existence would immediately ask for permission which would be bad UX, need to come up with a better solution
 
@@ -69,10 +70,11 @@ export class Cline {
 	api: ApiHandler
 	private terminalManager: TerminalManager
 	private urlContentFetcher: UrlContentFetcher
-	private browserSession: BrowserSession
+	browserSession: BrowserSession
 	private didEditFile: boolean = false
 	customInstructions?: string
 	autoApprovalSettings: AutoApprovalSettings
+	private browserSettings: BrowserSettings
 	apiConversationHistory: Anthropic.MessageParam[] = []
 	clineMessages: ClineMessage[] = []
 	private askResponse?: ClineAskResponse
@@ -107,6 +109,7 @@ export class Cline {
 		provider: ClineProvider,
 		apiConfiguration: ApiConfiguration,
 		autoApprovalSettings: AutoApprovalSettings,
+		browserSettings: BrowserSettings,
 		customInstructions?: string,
 		task?: string,
 		images?: string[],
@@ -116,10 +119,11 @@ export class Cline {
 		this.api = buildApiHandler(apiConfiguration)
 		this.terminalManager = new TerminalManager()
 		this.urlContentFetcher = new UrlContentFetcher(provider.context)
-		this.browserSession = new BrowserSession(provider.context)
+		this.browserSession = new BrowserSession(provider.context, browserSettings)
 		this.diffViewProvider = new DiffViewProvider(cwd)
 		this.customInstructions = customInstructions
 		this.autoApprovalSettings = autoApprovalSettings
+		this.browserSettings = browserSettings
 		if (historyItem) {
 			this.taskId = historyItem.id
 			this.conversationHistoryDeletedRange = historyItem.conversationHistoryDeletedRange
@@ -132,6 +136,11 @@ export class Cline {
 		}
 	}
 
+	updateBrowserSettings(browserSettings: BrowserSettings) {
+		this.browserSettings = browserSettings
+		this.browserSession.browserSettings = browserSettings
+	}
+
 	// Storing task to disk for history
 
 	private async ensureTaskDirectoryExists(): Promise<string> {
@@ -1177,7 +1186,12 @@ export class Cline {
 			throw new Error("MCP hub not available")
 		}
 
-		let systemPrompt = await SYSTEM_PROMPT(cwd, this.api.getModel().info.supportsComputerUse ?? false, mcpHub)
+		let systemPrompt = await SYSTEM_PROMPT(
+			cwd,
+			this.api.getModel().info.supportsComputerUse ?? false,
+			mcpHub,
+			this.browserSettings,
+		)
 		let settingsCustomInstructions = this.customInstructions?.trim()
 		const clineRulesFilePath = path.resolve(cwd, GlobalFileNames.clineRules)
 		let clineRulesFileInstructions: string | undefined

src/core/Cline.ts

@@ -2,11 +2,13 @@ import defaultShell from "default-shell"
 import os from "os"
 import osName from "os-name"
 import { McpHub } from "../../services/mcp/McpHub"
+import { BrowserSettings } from "../../shared/BrowserSettings"
 
 export const SYSTEM_PROMPT = async (
 	cwd: string,
 	supportsComputerUse: boolean,
 	mcpHub: McpHub,
+	browserSettings: BrowserSettings,
 ) => `You are Cline, a highly skilled software engineer with extensive knowledge in many programming languages, frameworks, design patterns, and best practices.
 
 ====
@@ -143,7 +145,7 @@ Usage:
 Description: Request to interact with a Puppeteer-controlled browser. Every action, except \`close\`, will be responded to with a screenshot of the browser's current state, along with any new console logs. You may only perform one browser action per message, and wait for the user's response including a screenshot and logs to determine the next action.
 - The sequence of actions **must always start with** launching the browser at a URL, and **must always end with** closing the browser. If you need to visit a new URL that is not possible to navigate to from the current webpage, you must first close the browser, then launch again at the new URL.
 - While the browser is active, only the \`browser_action\` tool can be used. No other tools should be called during this time. You may proceed to use other tools only after closing the browser. For example if you run into an error and need to fix a file, you must close the browser, then use other tools to make the necessary changes, then re-launch the browser to verify the result.
-- The browser window has a resolution of **900x600** pixels. When performing any click actions, ensure the coordinates are within this resolution range.
+- The browser window has a resolution of **${browserSettings.viewport.width}x${browserSettings.viewport.height}** pixels. When performing any click actions, ensure the coordinates are within this resolution range.
 - Before clicking on any elements such as icons, links, or buttons, you must consult the provided screenshot of the page to determine the coordinates of the element. The click should be targeted at the **center of the element**, not on its edges.
 Parameters:
 - action: (required) The action to perform. The available actions are:
@@ -161,7 +163,7 @@ Parameters:
         - Example: \`<action>close</action>\`
 - url: (optional) Use this for providing the URL for the \`launch\` action.
     * Example: <url>https://example.com</url>
-- coordinate: (optional) The X and Y coordinates for the \`click\` action. Coordinates should be within the **900x600** resolution.
+- coordinate: (optional) The X and Y coordinates for the \`click\` action. Coordinates should be within the **${browserSettings.viewport.width}x${browserSettings.viewport.height}** resolution.
     * Example: <coordinate>450,300</coordinate>
 - text: (optional) Use this for providing the text for the \`type\` action.
     * Example: <text>Hello, world!</text>

src/core/prompts/system.ts

@@ -23,6 +23,7 @@ import { openMention } from "../mentions"
 import { getNonce } from "./getNonce"
 import { getUri } from "./getUri"
 import { AutoApprovalSettings, DEFAULT_AUTO_APPROVAL_SETTINGS } from "../../shared/AutoApprovalSettings"
+import { BrowserSettings, DEFAULT_BROWSER_SETTINGS } from "../../shared/BrowserSettings"
 
 /*
 https://github.com/microsoft/vscode-webview-ui-toolkit-samples/blob/main/default/weather-webview/src/providers/WeatherViewProvider.ts
@@ -61,6 +62,7 @@ type GlobalStateKey =
 	| "openRouterModelId"
 	| "openRouterModelInfo"
 	| "autoApprovalSettings"
+	| "browserSettings"
 
 export const GlobalFileNames = {
 	apiConversationHistory: "api_conversation_history.json",
@@ -210,17 +212,18 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 
 	async initClineWithTask(task?: string, images?: string[]) {
 		await this.clearTask() // ensures that an exising task doesn't exist before starting a new one, although this shouldn't be possible since user must clear task before starting a new one
-		const { apiConfiguration, customInstructions, autoApprovalSettings } = await this.getState()
-		this.cline = new Cline(this, apiConfiguration, autoApprovalSettings, customInstructions, task, images)
+		const { apiConfiguration, customInstructions, autoApprovalSettings, browserSettings } = await this.getState()
+		this.cline = new Cline(this, apiConfiguration, autoApprovalSettings, browserSettings, customInstructions, task, images)
 	}
 
 	async initClineWithHistoryItem(historyItem: HistoryItem) {
 		await this.clearTask()
-		const { apiConfiguration, customInstructions, autoApprovalSettings } = await this.getState()
+		const { apiConfiguration, customInstructions, autoApprovalSettings, browserSettings } = await this.getState()
 		this.cline = new Cline(
 			this,
 			apiConfiguration,
 			autoApprovalSettings,
+			browserSettings,
 			customInstructions,
 			undefined,
 			undefined,
@@ -436,6 +439,20 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 							await this.postStateToWebview()
 						}
 						break
+					case "browserSettings":
+						if (message.browserSettings) {
+							await this.updateGlobalState("browserSettings", message.browserSettings)
+							if (this.cline) {
+								this.cline.updateBrowserSettings(message.browserSettings)
+							}
+							await this.postStateToWebview()
+						}
+						break
+					// case "relaunchChromeDebugMode":
+					// 	if (this.cline) {
+					// 		this.cline.browserSession.relaunchChromeDebugMode()
+					// 	}
+					// 	break
 					case "askResponse":
 						this.cline?.handleWebviewAskResponse(message.askResponse!, message.text, message.images)
 						break
@@ -908,8 +925,14 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 	}
 
 	async getStateToPostToWebview(): Promise<ExtensionState> {
-		const { apiConfiguration, lastShownAnnouncementId, customInstructions, taskHistory, autoApprovalSettings } =
-			await this.getState()
+		const {
+			apiConfiguration,
+			lastShownAnnouncementId,
+			customInstructions,
+			taskHistory,
+			autoApprovalSettings,
+			browserSettings,
+		} = await this.getState()
 		return {
 			version: this.context.extension?.packageJSON?.version ?? "",
 			apiConfiguration,
@@ -921,6 +944,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			taskHistory: (taskHistory || []).filter((item) => item.ts && item.task).sort((a, b) => b.ts - a.ts),
 			shouldShowAnnouncement: lastShownAnnouncementId !== this.latestAnnouncementId,
 			autoApprovalSettings,
+			browserSettings,
 		}
 	}
 
@@ -1006,6 +1030,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			customInstructions,
 			taskHistory,
 			autoApprovalSettings,
+			browserSettings,
 		] = await Promise.all([
 			this.getGlobalState("apiProvider") as Promise<ApiProvider | undefined>,
 			this.getGlobalState("apiModelId") as Promise<string | undefined>,
@@ -1036,6 +1061,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			this.getGlobalState("customInstructions") as Promise<string | undefined>,
 			this.getGlobalState("taskHistory") as Promise<HistoryItem[] | undefined>,
 			this.getGlobalState("autoApprovalSettings") as Promise<AutoApprovalSettings | undefined>,
+			this.getGlobalState("browserSettings") as Promise<BrowserSettings | undefined>,
 		])
 
 		let apiProvider: ApiProvider
@@ -1084,6 +1110,7 @@ export class ClineProvider implements vscode.WebviewViewProvider {
 			customInstructions,
 			taskHistory,
 			autoApprovalSettings: autoApprovalSettings || DEFAULT_AUTO_APPROVAL_SETTINGS, // default value can be 0 or empty string
+			browserSettings: browserSettings || DEFAULT_BROWSER_SETTINGS,
 		}
 	}
 

src/core/webview/ClineProvider.ts

@@ -8,20 +8,26 @@ import pWaitFor from "p-wait-for"
 import delay from "delay"
 import { fileExistsAtPath } from "../../utils/fs"
 import { BrowserActionResult } from "../../shared/ExtensionMessage"
+import { BrowserSettings } from "../../shared/BrowserSettings"
+// import * as chromeLauncher from "chrome-launcher"
 
 interface PCRStats {
 	puppeteer: { launch: typeof launch }
 	executablePath: string
 }
 
+// const DEBUG_PORT = 9222 // Chrome's default debugging port
+
 export class BrowserSession {
 	private context: vscode.ExtensionContext
 	private browser?: Browser
 	private page?: Page
 	private currentMousePosition?: string
+	browserSettings: BrowserSettings
 
-	constructor(context: vscode.ExtensionContext) {
+	constructor(context: vscode.ExtensionContext, browserSettings: BrowserSettings) {
 		this.context = context
+		this.browserSettings = browserSettings
 	}
 
 	private async ensureChromiumExists(): Promise<PCRStats> {
@@ -45,6 +51,70 @@ export class BrowserSession {
 		return stats
 	}
 
+	// private async checkExistingChromeDebugger(): Promise<boolean> {
+	// 	try {
+	// 		// Try to connect to existing debugger
+	// 		const response = await fetch(`http://localhost:${DEBUG_PORT}/json/version`)
+	// 		return response.ok
+	// 	} catch {
+	// 		return false
+	// 	}
+	// }
+
+	// async relaunchChromeDebugMode() {
+	// 	const result = await vscode.window.showWarningMessage(
+	// 		"This will close your existing Chrome tabs and relaunch Chrome in debug mode. Are you sure?",
+	// 		{ modal: true },
+	// 		"Yes",
+	// 	)
+
+	// 	if (result !== "Yes") {
+	// 		return
+	// 	}
+
+	// 	// // Kill any existing Chrome instances
+	// 	// await chromeLauncher.killAll()
+
+	// 	// // Launch Chrome with debug port
+	// 	// const launcher = new chromeLauncher.Launcher({
+	// 	// 	port: DEBUG_PORT,
+	// 	// 	chromeFlags: ["--remote-debugging-port=" + DEBUG_PORT, "--no-first-run", "--no-default-browser-check"],
+	// 	// })
+
+	// 	// await launcher.launch()
+	// 	const installation = chromeLauncher.Launcher.getFirstInstallation()
+	// 	if (!installation) {
+	// 		throw new Error("Could not find Chrome installation on this system")
+	// 	}
+	// 	console.log("chrome installation", installation)
+	// }
+
+	// private async getSystemChromeExecutablePath(): Promise<string> {
+	// 	// Find installed Chrome
+	// 	const installation = chromeLauncher.Launcher.getFirstInstallation()
+	// 	if (!installation) {
+	// 		throw new Error("Could not find Chrome installation on this system")
+	// 	}
+	// 	console.log("chrome installation", installation)
+	// 	return installation
+	// }
+
+	// /**
+	//  * Helper to detect user’s default Chrome data dir.
+	//  * Adjust for OS if needed.
+	//  */
+	// private getDefaultChromeUserDataDir(): string {
+	// 	const homedir = require("os").homedir()
+	// 	switch (process.platform) {
+	// 		case "win32":
+	// 			return path.join(homedir, "AppData", "Local", "Google", "Chrome", "User Data")
+	// 		case "darwin":
+	// 			return path.join(homedir, "Library", "Application Support", "Google", "Chrome")
+	// 		default:
+	// 			return path.join(homedir, ".config", "google-chrome")
+	// 	}
+	// }
+
 	async launchBrowser() {
 		console.log("launch browser called")
 		if (this.browser) {
@@ -58,12 +128,29 @@ export class BrowserSession {
 				"--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
 			],
 			executablePath: stats.executablePath,
-			defaultViewport: {
-				width: 900,
-				height: 600,
-			},
-			// headless: false,
+			defaultViewport: this.browserSettings.viewport,
+			headless: this.browserSettings.headless,
 		})
+
+		// if (this.browserSettings.chromeType === "system") {
+		// 	const userDataDir = this.getDefaultChromeUserDataDir()
+		// 	this.browser = await stats.puppeteer.launch({
+		// 		args: [`--user-data-dir=${userDataDir}`, "--profile-directory=Default"],
+		// 		executablePath: await this.getSystemChromeExecutablePath(),
+		// 		defaultViewport: this.browserSettings.viewport,
+		// 		headless: this.browserSettings.headless,
+		// 	})
+		// } else {
+		// 	this.browser = await stats.puppeteer.launch({
+		// 		args: [
+		// 			"--user-agent=Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/128.0.0.0 Safari/537.36",
+		// 		],
+		// 		executablePath: stats.executablePath,
+		// 		defaultViewport: this.browserSettings.viewport,
+		// 		headless: this.browserSettings.headless,
+		// 	})
+		// }
+
 		// (latest version of puppeteer does not add headless to user agent)
 		this.page = await this.browser?.newPage()
 	}

src/services/browser/BrowserSession.ts

@@ -0,0 +1,27 @@
+export interface BrowserSettings {
+	// Viewport size settings
+	viewport: {
+		width: number
+		height: number
+	}
+	// Browser mode settings
+	headless: boolean
+	// Chrome installation to use
+	// chromeType: "chromium" | "system"
+}
+
+export const DEFAULT_BROWSER_SETTINGS: BrowserSettings = {
+	viewport: {
+		width: 900,
+		height: 600,
+	},
+	headless: true,
+	// chromeType: "chromium",
+}
+
+export const BROWSER_VIEWPORT_PRESETS = {
+	"Large Desktop (1280x800)": { width: 1280, height: 800 },
+	"Small Desktop (900x600)": { width: 900, height: 600 },
+	"Tablet (768x1024)": { width: 768, height: 1024 },
+	"Mobile (360x640)": { width: 360, height: 640 },
+} as const