feat: add mouse action and improve descriptions for actions prompt (#79)

* feat: add mouse action and improve descriptions for actions prompt

* fix: add Clear, remove Check, update schema and examples

* fix: remove Check action

Author: mmmay0722

webqa_agent/actions/action_executor.py

@@ -14,14 +14,13 @@ def __init__(self, action_handler):
             "Clear": self._execute_clear,
             "Scroll": self._execute_scroll,
             "KeyboardPress": self._execute_keyboard_press,
-            "FalsyConditionStatement": self._execute_falsy,
-            "Check": self._execute_check,
             "GetNewPage": self._execute_get_new_page,
             "Upload": self._execute_upload,
             "SelectDropdown": self._execute_select_dropdown,
             "Drag": self._execute_drag,
             "GoToPage": self._execute_go_to_page,  # Added missing action
             "GoBack": self._execute_go_back,  # Added browser back navigation
+            "Mouse": self._execute_mouse, # Added mouse action
         }
 
     async def initialize(self):
@@ -146,14 +145,6 @@ async def _execute_keyboard_press(self, action):
         else:
             return {"success": False, "message": "Keyboard press failed."}
 
-    async def _execute_falsy(self, action):
-        """Execute falsy condition statement."""
-        return {"success": True, "message": "Falsy condition met."}
-
-    async def _execute_check(self, action):
-        """Execute check action."""
-        return {"success": True, "message": "Check action completed."}
-
     async def _execute_get_new_page(self, action):
         """Execute get new page action."""
         success = await self._actions.get_new_page()
@@ -335,4 +326,68 @@ async def _execute_go_back(self, action):
                 return {"success": False, "message": "Go back action not supported by action handler"}
         except Exception as e:
             logging.error(f"Go back action failed: {str(e)}")
-            return {"success": False, "message": f"Go back failed: {str(e)}", "playwright_error": str(e)}
+            return {"success": False, "message": f"Go back failed: {str(e)}", "playwright_error": str(e)}
+    
+    async def _execute_mouse(self, action):
+        """Unified mouse action supporting move and wheel.
+
+        Accepted param formats:
+        - { op: "move", x: number, y: number }
+        - { op: "wheel", deltaX: number, deltaY: number }
+        - Back-compat: if op is omitted, decide by presence of keys
+        """
+        try:
+            param = action.get("param")
+            if not param or not isinstance(param, dict):
+                return {"success": False, "message": "Missing or invalid param for mouse action"}
+            
+            op = param.get("op")
+            
+            # Auto-detect if op not provided or empty
+            if not op:
+                if "x" in param and "y" in param:
+                    op = "move"
+                elif "deltaX" in param or "deltaY" in param:
+                    op = "wheel"
+                else:
+                    return {"success": False, "message": "Missing mouse operation parameters (x/y or deltaX/deltaY)"}
+
+            if op == "move":
+                if not self._validate_params(action, ["param.x", "param.y"]):
+                    return {"success": False, "message": "Missing x or y coordinates for mouse move"}
+                
+                x = param.get("x")
+                y = param.get("y")
+                
+                # Validate coordinates are numbers
+                if not isinstance(x, (int, float)) or not isinstance(y, (int, float)):
+                    return {"success": False, "message": "x and y coordinates must be numbers"}
+                
+                success = await self._actions.mouse_move(x, y)
+                if success:
+                    return {"success": True, "message": f"Mouse moved to ({x}, {y})"}
+                else:
+                    return {"success": False, "message": "Mouse move action failed"}
+
+            elif op == "wheel":
+                # Default missing keys to 0
+                dx = param.get("deltaX", 0)
+                dy = param.get("deltaY", 0)
+                
+                # Validate deltas are numbers
+                if not isinstance(dx, (int, float)) or not isinstance(dy, (int, float)):
+                    return {"success": False, "message": "deltaX and deltaY must be numbers"}
+                
+                success = await self._actions.mouse_wheel(dx, dy)
+                if success:
+                    return {"success": True, "message": f"Mouse wheel scrolled (deltaX: {dx}, deltaY: {dy})"}
+                else:
+                    return {"success": False, "message": "Mouse wheel action failed"}
+
+            else:
+                logging.error(f"Unknown mouse op: {op}. Expected 'move' or 'wheel'.")
+                return {"success": False, "message": f"Unknown mouse operation: {op}. Expected 'move' or 'wheel'"}
+                
+        except Exception as e:
+            logging.error(f"Mouse action execution failed: {str(e)}")
+            return {"success": False, "message": f"Mouse action failed with an exception: {e}"}

webqa_agent/actions/action_handler.py

@@ -1429,3 +1429,30 @@ async def drag(self, source_coords, target_coords):
         except Exception as e:
             logging.error(f'Drag action failed: {str(e)}')
             return False
+    
+    async def mouse_move(self, x: int | float, y: int | float) -> bool:
+        """Move mouse to absolute coordinates (x, y)."""
+        try:
+            # Coerce to numbers in case strings are provided
+            target_x = float(x)
+            target_y = float(y)
+            await self.page.mouse.move(target_x, target_y)
+            logging.info(f"mouse move to ({target_x}, {target_y})")
+            await asyncio.sleep(0.1)
+            return True
+        except Exception as e:
+            logging.error(f"Mouse move failed: {str(e)}")
+            return False
+
+    async def mouse_wheel(self, delta_x: int | float = 0, delta_y: int | float = 0) -> bool:
+        """Scroll the mouse wheel by delta values."""
+        try:
+            dx = float(delta_x) if delta_x is not None else 0.0
+            dy = float(delta_y) if delta_y is not None else 0.0
+            await self.page.mouse.wheel(dx, dy)
+            logging.info(f"mouse wheel by (deltaX={dx}, deltaY={dy})")
+            await asyncio.sleep(0.1)
+            return True
+        except Exception as e:
+            logging.error(f"Mouse wheel failed: {str(e)}")
+            return False

webqa_agent/llm/prompt.py

@@ -51,7 +51,7 @@ class LLMPrompt:
 
     ## Anchor Usage Rule
     Anchors are strictly used for reference during disambiguation.
-    **NEVER** interact (Tap/Hover/Check) with anchor elements directly.
+    **NEVER** interact (Tap/Hover) with anchor elements directly.
 
     ## Scroll Behavior Constraints
     - Avoid planning `Scroll` if the page is already at the bottom.
@@ -97,6 +97,8 @@ class LLMPrompt:
         * `value` is the final required input value based on the existing input. No matter what modifications are required, just provide the final value to replace the existing input value.
         * For Input actions, if the page or validation message requires a minimum length, the value you generate MUST strictly meet or exceed this length. For Chinese, count each character as 1.
         * `clear_before_type`: Set to `true` if the instruction explicitly says to 'clear' the field before typing, or if you are correcting a previous failed input. Defaults to `false`.
+        - type: 'Clear', clear the content of an input field
+        * {{ locate: {{ id: string }}, param: null }}
         - type: 'KeyboardPress', press a key
         * {{ param: {{ value: string }} }}
         - type: 'Upload', upload a file (or click the upload button)
@@ -124,9 +126,6 @@ class LLMPrompt:
         * use this action when you need to go back to the previous page in the browser history, similar to clicking the browser's back button.
         - type: 'Sleep'
         * {{ param: {{ timeMs: number }} }}
-        - type: 'Check'
-        * {{ param: null }}
-        * use this action when the instruction is a "check" or "verify" or "validate" statement.
         - type: 'Drag', drag an slider or element from source to target position
           For Drag action, use the following format:
             {
@@ -152,6 +151,21 @@ class LLMPrompt:
         * selection_path is the text of the option to be selected.
         * if the selection_path is a string, it means the option is the first level of the dropdown.
         * if the selection_path is a list, it means the option is the nth level of the dropdown.
+        - type: 'Mouse', unified mouse action for move and wheel
+         {
+           "param": {
+             "op": 'move' | 'wheel',
+             // move operation
+             "x"?: number,
+             "y"?: number,
+             // wheel operation
+             "deltaX"?: number,
+             "deltaY"?: number
+           },
+           "locate": null
+         }
+        * When op is omitted, auto-detect by provided fields: x+y => move; deltaX/deltaY => wheel.
+
 
     ## Further Plan Format
     If the task isn't completed:
@@ -178,13 +192,19 @@ class LLMPrompt:
 
     ### Supported Actions:
     - Tap: Click on a specified page element (such as a button or link). Typically used to trigger a click event.
+    - Hover: Move the mouse over a specified page element (such as a button or link). Typically used to show tooltip or hover effect.
     - Scroll: Scroll the page or a specific region. You can specify the direction (down, up), the scroll distance, or scroll to the edge of the page/region.
     - Input: Enter text into an input field or textarea. This action will replace the current value with the specified final value.
+    - Clear: Clear the content of an input field. Requires the input's external id in locate.
     - Sleep: Wait for a specified amount of time (in milliseconds). Useful for waiting for page loads or asynchronous content to render.
     - Upload: Upload a file
     - KeyboardPress: Simulate a keyboard key press, such as Enter, Tab, or arrow keys.
     - Drag: Perform a drag-and-drop operation. Moves the mouse from a starting coordinate to a target coordinate, often used for sliders, sorting, or drag-and-drop interfaces. Requires both source and target coordinates.
     - SelectDropdown: Select an option from a dropdown menu which is user's expected option. The dropdown element is the first level of the dropdown menu. IF You can see the dropdown element, you cannot click the dropdown element, you should directly select the option.
+    - GoToPage: Navigate directly to a specific URL. Useful for returning to the homepage, navigating to known pages, or entering a new web address. Requires a URL parameter.
+    - GoBack: Navigate back to the previous page in the browser history, similar to clicking the browser's back button. Does not require any parameters.
+    - GetNewPage: Get the new page or open in new tab or open in new window. Use this action when the previous action (e.g., clicking a link that opens in a new tab) creates a new browser context that needs to be accessed.
+    - Mouse: Unified mouse action for move and wheel.
 
     Please ensure the output is a valid **JSON** object. Do **not** include any markdown, backticks, or code block indicators.
 
@@ -193,7 +213,7 @@ class LLMPrompt:
           "actions": [
             {
               "thought": "Reasoning for this action and why it's feasible on the current page.",
-              "type": "Tap" | "Scroll" | "Input" | "Sleep" | "Check" | "Upload" | "KeyboardPress" | "Drag" | "SelectDropdown" | "GoToPage" | "GoBack",
+              "type": "Tap" | "Hover" | "Scroll" | "Input" | "Clear" | "Sleep" | "Upload" | "KeyboardPress" | "Drag" | "SelectDropdown" | "GoToPage" | "GoBack" | "GetNewPage" | "Mouse",
               "param": {...} | null,
               "locate": {...} | null
             }
@@ -216,13 +236,12 @@ class LLMPrompt:
           - If mismatched: `targetVerified: false` and include error: "Planned element does not match the user's expected target"
         - If an expected element is not found on the page:
           - For imperative instruction: return `error` and empty actions.
-          - For tolerant instructions like "If popup exists, close it", return `FalsyConditionStatement` action.
 
         ---
 
         ### Unified Few-shot Examples
 
-        #### Example 1: Tap + Sleep + Check (task incomplete)
+        #### Example 1: Tap + Sleep (task incomplete)
         "Click send button and wait 50s"
 
         ====================
@@ -256,7 +275,7 @@ class LLMPrompt:
         }
         ```
 
-        #### Example 2: Scroll + Check (scroll history aware)
+        #### Example 2: Scroll (scroll history aware)
         ```json
         {
           "actions": [