"Add Element to Chat" in integrated browser (#286649)

* "Add element to chat" in integrated browser

* Support disabling AI features, PR feedback

* PR feedback

---------

Co-authored-by: Joaquín Ruales <1588988+jruales@users.noreply.github.com>

Author: kycutler

src/vs/platform/browserElements/common/browserElements.ts

@@ -15,20 +15,52 @@ export interface IElementData {
 	readonly bounds: IRectangle;
 }
 
-export enum BrowserType {
-	SimpleBrowser = 'simpleBrowser',
-	LiveServer = 'liveServer',
+/**
+ * Locator for identifying a browser target/webview.
+ * Uses either the parent webview or browser view id to uniquely identify the target.
+ */
+export interface IBrowserTargetLocator {
+	/**
+	 * Identifier of the parent webview hosting the target.
+	 *
+	 * Exactly one of {@link webviewId} or {@link browserViewId} should be provided.
+	 * Use this when the target is rendered inside a webview.
+	 */
+	readonly webviewId?: string;
+	/**
+	 * Identifier of the browser view hosting the target.
+	 *
+	 * Exactly one of {@link webviewId} or {@link browserViewId} should be provided.
+	 * Use this when the target is rendered inside a browser view rather than a webview.
+	 */
+	readonly browserViewId?: string;
 }
 
-
 export interface INativeBrowserElementsService {
 
 	readonly _serviceBrand: undefined;
 
 	// Properties
 	readonly windowId: number;
 
-	getElementData(rect: IRectangle, token: CancellationToken, browserType: BrowserType, cancellationId?: number): Promise<IElementData | undefined>;
+	getElementData(rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator, cancellationId?: number): Promise<IElementData | undefined>;
+
+	startDebugSession(token: CancellationToken, locator: IBrowserTargetLocator, cancelAndDetachId?: number): Promise<void>;
+}
+
+/**
+ * Extract a display name from outer HTML (e.g., "div#myId.myClass1.myClass2")
+ */
+export function getDisplayNameFromOuterHTML(outerHTML: string): string {
+	const firstElementMatch = outerHTML.match(/^<([^ >]+)([^>]*?)>/);
+	if (!firstElementMatch) {
+		throw new Error('No outer element found');
+	}
 
-	startDebugSession(token: CancellationToken, browserType: BrowserType, cancelAndDetachId?: number): Promise<void>;
+	const tagName = firstElementMatch[1];
+	const idMatch = firstElementMatch[2].match(/\s+id\s*=\s*["']([^"']+)["']/i);
+	const id = idMatch ? `#${idMatch[1]}` : '';
+	const classMatch = firstElementMatch[2].match(/\s+class\s*=\s*["']([^"']+)["']/i);
+	const className = classMatch ? `.${classMatch[1].replace(/\s+/g, '.')}` : '';
+	return `${tagName}${id}${className}`;
 }

src/vs/platform/browserElements/electron-main/nativeBrowserElementsMainService.ts

@@ -3,7 +3,7 @@
  *  Licensed under the MIT License. See License.txt in the project root for license information.
  *--------------------------------------------------------------------------------------------*/
 
-import { BrowserType, IElementData, INativeBrowserElementsService } from '../common/browserElements.js';
+import { IElementData, INativeBrowserElementsService, IBrowserTargetLocator } from '../common/browserElements.js';
 import { CancellationToken } from '../../../base/common/cancellation.js';
 import { IRectangle } from '../../window/common/window.js';
 import { BrowserWindow, webContents } from 'electron';
@@ -14,6 +14,7 @@ import { IWindowsMainService } from '../../windows/electron-main/windows.js';
 import { createDecorator } from '../../instantiation/common/instantiation.js';
 import { Disposable } from '../../../base/common/lifecycle.js';
 import { AddFirstParameterToFunctions } from '../../../base/common/types.js';
+import { IBrowserViewMainService } from '../../browserView/electron-main/browserViewMainService.js';
 
 export const INativeBrowserElementsMainService = createDecorator<INativeBrowserElementsMainService>('browserElementsMainService');
 export interface INativeBrowserElementsMainService extends AddFirstParameterToFunctions<INativeBrowserElementsService, Promise<unknown> /* only methods, not events */, number | undefined /* window ID */> { }
@@ -27,89 +28,78 @@ interface NodeDataResponse {
 export class NativeBrowserElementsMainService extends Disposable implements INativeBrowserElementsMainService {
 	_serviceBrand: undefined;
 
-	currentLocalAddress: string | undefined;
-
 	constructor(
 		@IWindowsMainService private readonly windowsMainService: IWindowsMainService,
 		@IAuxiliaryWindowsMainService private readonly auxiliaryWindowsMainService: IAuxiliaryWindowsMainService,
-
+		@IBrowserViewMainService private readonly browserViewMainService: IBrowserViewMainService
 	) {
 		super();
 	}
 
 	get windowId(): never { throw new Error('Not implemented in electron-main'); }
 
-	async findWebviewTarget(debuggers: Electron.Debugger, windowId: number, browserType: BrowserType): Promise<string | undefined> {
+	/**
+	 * Find the webview target that matches the given locator.
+	 * Checks either webviewId or browserViewId depending on what's provided.
+	 */
+	async findWebviewTarget(debuggers: Electron.Debugger, locator: IBrowserTargetLocator): Promise<string | undefined> {
 		const { targetInfos } = await debuggers.sendCommand('Target.getTargets');
-		let target: typeof targetInfos[number] | undefined = undefined;
-		const matchingTarget = targetInfos.find((targetInfo: { url: string }) => {
-			try {
-				const url = new URL(targetInfo.url);
-				if (browserType === BrowserType.LiveServer) {
-					return url.searchParams.get('id') && url.searchParams.get('extensionId') === 'ms-vscode.live-server';
-				} else if (browserType === BrowserType.SimpleBrowser) {
-					return url.searchParams.get('parentId') === windowId.toString() && url.searchParams.get('extensionId') === 'vscode.simple-browser';
+
+		if (locator.webviewId) {
+			let extensionId = '';
+			for (const targetInfo of targetInfos) {
+				try {
+					const url = new URL(targetInfo.url);
+					if (url.searchParams.get('id') === locator.webviewId) {
+						extensionId = url.searchParams.get('extensionId') || '';
+						break;
+					}
+				} catch (err) {
+					// ignore
 				}
-				return false;
-			} catch (err) {
-				return false;
 			}
-		});
-
-		// search for webview via search parameters
-		if (matchingTarget) {
-			let resultId: string | undefined;
-			let url: URL | undefined;
-			try {
-				url = new URL(matchingTarget.url);
-				resultId = url.searchParams.get('id')!;
-			} catch (e) {
+			if (!extensionId) {
 				return undefined;
 			}
 
-			target = targetInfos.find((targetInfo: { url: string }) => {
+			// search for webview via search parameters
+			const target = targetInfos.find((targetInfo: { url: string }) => {
 				try {
 					const url = new URL(targetInfo.url);
-					const isLiveServer = browserType === BrowserType.LiveServer && url.searchParams.get('serverWindowId') === resultId;
-					const isSimpleBrowser = browserType === BrowserType.SimpleBrowser && url.searchParams.get('id') === resultId && url.searchParams.has('vscodeBrowserReqId');
+					const isLiveServer = extensionId === 'ms-vscode.live-server' && url.searchParams.get('serverWindowId') === locator.webviewId;
+					const isSimpleBrowser = extensionId === 'vscode.simple-browser' && url.searchParams.get('id') === locator.webviewId && url.searchParams.has('vscodeBrowserReqId');
 					if (isLiveServer || isSimpleBrowser) {
-						this.currentLocalAddress = url.origin;
 						return true;
 					}
 					return false;
 				} catch (e) {
 					return false;
 				}
 			});
-
-			if (target) {
-				return target.targetId;
-			}
+			return target?.targetId;
 		}
 
-		// fallback: search for webview without parameters based on current origin
-		target = targetInfos.find((targetInfo: { url: string }) => {
-			try {
-				const url = new URL(targetInfo.url);
-				return (this.currentLocalAddress === url.origin);
-			} catch (e) {
-				return false;
-			}
-		});
+		if (locator.browserViewId) {
+			const webContentsInstance = this.browserViewMainService.tryGetBrowserView(locator.browserViewId)?.webContents;
+			const target = targetInfos.find((targetInfo: { targetId: string; type: string }) => {
+				if (targetInfo.type !== 'page') {
+					return false;
+				}
 
-		if (!target) {
-			return undefined;
+				return webContents.fromDevToolsTargetId(targetInfo.targetId) === webContentsInstance;
+			});
+			return target?.targetId;
 		}
 
-		return target.targetId;
+		return undefined;
 	}
 
-	async waitForWebviewTargets(debuggers: Electron.Debugger, windowId: number, browserType: BrowserType): Promise<string | undefined> {
+	async waitForWebviewTargets(debuggers: Electron.Debugger, locator: IBrowserTargetLocator): Promise<string | undefined> {
 		const start = Date.now();
 		const timeout = 10000;
 
 		while (Date.now() - start < timeout) {
-			const targetId = await this.findWebviewTarget(debuggers, windowId, browserType);
+			const targetId = await this.findWebviewTarget(debuggers, locator);
 			if (targetId) {
 				return targetId;
 			}
@@ -122,7 +112,7 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 		return undefined;
 	}
 
-	async startDebugSession(windowId: number | undefined, token: CancellationToken, browserType: BrowserType, cancelAndDetachId?: number): Promise<void> {
+	async startDebugSession(windowId: number | undefined, token: CancellationToken, locator: IBrowserTargetLocator, cancelAndDetachId?: number): Promise<void> {
 		const window = this.windowById(windowId);
 		if (!window?.win) {
 			return undefined;
@@ -142,7 +132,7 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 		}
 
 		try {
-			const matchingTargetId = await this.waitForWebviewTargets(debuggers, windowId!, browserType);
+			const matchingTargetId = await this.waitForWebviewTargets(debuggers, locator);
 			if (!matchingTargetId) {
 				if (debuggers.isAttached()) {
 					debuggers.detach();
@@ -187,7 +177,7 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 		}
 	}
 
-	async getElementData(windowId: number | undefined, rect: IRectangle, token: CancellationToken, browserType: BrowserType, cancellationId?: number): Promise<IElementData | undefined> {
+	async getElementData(windowId: number | undefined, rect: IRectangle, token: CancellationToken, locator: IBrowserTargetLocator, cancellationId?: number): Promise<IElementData | undefined> {
 		const window = this.windowById(windowId);
 		if (!window?.win) {
 			return undefined;
@@ -208,7 +198,7 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 
 		let targetSessionId: string | undefined = undefined;
 		try {
-			const targetId = await this.findWebviewTarget(debuggers, windowId!, browserType);
+			const targetId = await this.findWebviewTarget(debuggers, locator);
 			const { sessionId } = await debuggers.sendCommand('Target.attachToTarget', {
 				targetId: targetId,
 				flatten: true,
@@ -373,7 +363,7 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 						const content = model.content;
 						const margin = model.margin;
 						const x = Math.min(margin[0], content[0]);
-						const y = Math.min(margin[1], content[1]) + 32.4; // 32.4 is height of the title bar
+						const y = Math.min(margin[1], content[1]);
 						const width = Math.max(margin[2] - margin[0], content[2] - content[0]);
 						const height = Math.max(margin[5] - margin[1], content[5] - content[1]);
 
@@ -416,7 +406,7 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 		});
 	}
 
-	formatMatchedStyles(matched: { inlineStyle?: { cssProperties?: Array<{ name: string; value: string }> }; matchedCSSRules?: Array<{ rule: { selectorList: { selectors: Array<{ text: string }> }; origin: string; style: { cssProperties: Array<{ name: string; value: string }> } } }>; inherited?: Array<{ matchedCSSRules?: Array<{ rule: { selectorList: { selectors: Array<{ text: string }> }; origin: string; style: { cssProperties: Array<{ name: string; value: string }> } } }> }> }): string {
+	formatMatchedStyles(matched: { inlineStyle?: { cssProperties?: Array<{ name: string; value: string }> }; matchedCSSRules?: Array<{ rule: { selectorList: { selectors: Array<{ text: string }> }; origin: string; style: { cssProperties: Array<{ name: string; value: string }> } } }>; inherited?: Array<{ inlineStyle?: { cssText: string }; matchedCSSRules?: Array<{ rule: { selectorList: { selectors: Array<{ text: string }> }; origin: string; style: { cssProperties: Array<{ name: string; value: string }> } } }> }> }): string {
 		const lines: string[] = [];
 
 		// inline
@@ -451,6 +441,14 @@ export class NativeBrowserElementsMainService extends Disposable implements INat
 		if (matched.inherited?.length) {
 			let level = 1;
 			for (const inherited of matched.inherited) {
+				const inline = inherited.inlineStyle;
+				if (inline) {
+					lines.push(`/* Inherited from ancestor level ${level} (inline) */`);
+					lines.push('element {');
+					lines.push(inline.cssText);
+					lines.push('}\n');
+				}
+
 				const rules = inherited.matchedCSSRules || [];
 				for (const ruleEntry of rules) {
 					const rule = ruleEntry.rule;

src/vs/platform/browserView/electron-main/browserView.ts

@@ -271,6 +271,10 @@ export class BrowserView extends Disposable {
 		});
 	}
 
+	get webContents(): Electron.WebContents {
+		return this._view.webContents;
+	}
+
 	/**
 	 * Get the current state of this browser view
 	 */

src/vs/platform/browserView/electron-main/browserViewMainService.ts

@@ -16,7 +16,7 @@ import { generateUuid } from '../../../base/common/uuid.js';
 export const IBrowserViewMainService = createDecorator<IBrowserViewMainService>('browserViewMainService');
 
 export interface IBrowserViewMainService extends IBrowserViewService {
-	// Additional electron-specific methods can be added here if needed in the future
+	tryGetBrowserView(id: string): BrowserView | undefined;
 }
 
 // Same as webviews
@@ -96,6 +96,10 @@ export class BrowserViewMainService extends Disposable implements IBrowserViewMa
 		return view.getState();
 	}
 
+	tryGetBrowserView(id: string): BrowserView | undefined {
+		return this.browserViews.get(id);
+	}
+
 	/**
 	 * Get a browser view or throw if not found
 	 */