diff --git a/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap b/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap index e9538bc308b..5e49f384b14 100644 --- a/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap +++ b/src/core/prompts/__tests__/__snapshots__/system.test.ts.snap @@ -2714,6 +2714,9 @@ Parameters: * launch: Launch a new Puppeteer-controlled browser instance at the specified URL. This **must always be the first action**. - Use with the \`url\` parameter to provide the URL. - Ensure the URL is valid and includes the appropriate protocol (e.g. http://localhost:3000/page, file:///path/to/file.html, etc.) + * hover: Move the cursor to a specific x,y coordinate. + - Use with the \`coordinate\` parameter to specify the location. + - Always move to the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. * click: Click at a specific x,y coordinate. - Use with the \`coordinate\` parameter to specify the location. - Always click in the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. @@ -2725,7 +2728,7 @@ Parameters: - Example: \`close\` - url: (optional) Use this for providing the URL for the \`launch\` action. * Example: https://example.com -- coordinate: (optional) The X and Y coordinates for the \`click\` action. Coordinates should be within the **900x600** resolution. +- coordinate: (optional) The X and Y coordinates for the \`click\` and \`hover\` actions. Coordinates should be within the **900x600** resolution. * Example: 450,300 - text: (optional) Use this for providing the text for the \`type\` action. * Example: Hello, world! @@ -3625,6 +3628,9 @@ Parameters: * launch: Launch a new Puppeteer-controlled browser instance at the specified URL. This **must always be the first action**. - Use with the \`url\` parameter to provide the URL. - Ensure the URL is valid and includes the appropriate protocol (e.g. http://localhost:3000/page, file:///path/to/file.html, etc.) + * hover: Move the cursor to a specific x,y coordinate. + - Use with the \`coordinate\` parameter to specify the location. + - Always move to the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. * click: Click at a specific x,y coordinate. - Use with the \`coordinate\` parameter to specify the location. - Always click in the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. @@ -3636,7 +3642,7 @@ Parameters: - Example: \`close\` - url: (optional) Use this for providing the URL for the \`launch\` action. * Example: https://example.com -- coordinate: (optional) The X and Y coordinates for the \`click\` action. Coordinates should be within the **1280x800** resolution. +- coordinate: (optional) The X and Y coordinates for the \`click\` and \`hover\` actions. Coordinates should be within the **1280x800** resolution. * Example: 450,300 - text: (optional) Use this for providing the text for the \`type\` action. * Example: Hello, world! diff --git a/src/core/prompts/tools/browser-action.ts b/src/core/prompts/tools/browser-action.ts index 9b5f1c4ee82..bf6ee8c2338 100644 --- a/src/core/prompts/tools/browser-action.ts +++ b/src/core/prompts/tools/browser-action.ts @@ -15,6 +15,9 @@ Parameters: * launch: Launch a new Puppeteer-controlled browser instance at the specified URL. This **must always be the first action**. - Use with the \`url\` parameter to provide the URL. - Ensure the URL is valid and includes the appropriate protocol (e.g. http://localhost:3000/page, file:///path/to/file.html, etc.) + * hover: Move the cursor to a specific x,y coordinate. + - Use with the \`coordinate\` parameter to specify the location. + - Always move to the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. * click: Click at a specific x,y coordinate. - Use with the \`coordinate\` parameter to specify the location. - Always click in the center of an element (icon, button, link, etc.) based on coordinates derived from a screenshot. @@ -26,7 +29,7 @@ Parameters: - Example: \`close\` - url: (optional) Use this for providing the URL for the \`launch\` action. * Example: https://example.com -- coordinate: (optional) The X and Y coordinates for the \`click\` action. Coordinates should be within the **${args.browserViewportSize}** resolution. +- coordinate: (optional) The X and Y coordinates for the \`click\` and \`hover\` actions. Coordinates should be within the **${args.browserViewportSize}** resolution. * Example: 450,300 - text: (optional) Use this for providing the text for the \`type\` action. * Example: Hello, world! diff --git a/src/core/tools/browserActionTool.ts b/src/core/tools/browserActionTool.ts index 8a9051070d8..cf1d417b5f5 100644 --- a/src/core/tools/browserActionTool.ts +++ b/src/core/tools/browserActionTool.ts @@ -72,7 +72,7 @@ export async function browserActionTool( await cline.browserSession.launchBrowser() browserActionResult = await cline.browserSession.navigateToUrl(url) } else { - if (action === "click") { + if (action === "click" || action === "hover") { if (!coordinate) { cline.consecutiveMistakeCount++ pushToolResult(await cline.sayAndCreateMissingParamError("browser_action", "coordinate")) @@ -103,6 +103,9 @@ export async function browserActionTool( case "click": browserActionResult = await cline.browserSession.click(coordinate!) break + case "hover": + browserActionResult = await cline.browserSession.hover(coordinate!) + break case "type": browserActionResult = await cline.browserSession.type(text!) break @@ -121,6 +124,7 @@ export async function browserActionTool( switch (action) { case "launch": case "click": + case "hover": case "type": case "scroll_down": case "scroll_up":