Merge pull request #12 from vlad-terin/mousedragopenapp

Mousedragopenapp

Author: vlad-terin

src/action_handlers.py

@@ -3,6 +3,8 @@
 import base64
 import os
 import sys
+import subprocess
+import time
 
 import mcp.types as types
 # Import vnc_client from the current directory
@@ -476,6 +478,181 @@ def handle_remote_macos_mouse_move(arguments: dict[str, Any]) -> list[types.Text
 Target dimensions: {target_width}x{target_height}
 Scale factors: {scale_factors['x']:.4f}x, {scale_factors['y']:.4f}y"""
         )]
+    finally:
+        # Close VNC connection
+        vnc.close()
+
+
+def handle_remote_macos_open_application(arguments: dict[str, Any]) -> List[types.TextContent]:
+    """
+    Opens or activates an application on the remote MacOS machine using VNC.
+
+    Args:
+        arguments: Dictionary containing:
+            - identifier: App name, path, or bundle ID
+
+    Returns:
+        List containing a TextContent with the result
+    """
+    # Use environment variables
+    host = MACOS_HOST
+    port = MACOS_PORT
+    password = MACOS_PASSWORD
+    username = MACOS_USERNAME
+    encryption = VNC_ENCRYPTION
+
+    identifier = arguments.get("identifier")
+    if not identifier:
+        raise ValueError("identifier is required")
+
+    start_time = time.time()
+
+    # Initialize VNC client
+    vnc = VNCClient(host=host, port=port, password=password, username=username, encryption=encryption)
+
+    # Connect to remote MacOs machine
+    success, error_message = vnc.connect()
+    if not success:
+        error_msg = f"Failed to connect to remote MacOs machine at {host}:{port}. {error_message}"
+        return [types.TextContent(type="text", text=error_msg)]
+
+    try:
+        # Send Command+Space to open Spotlight
+        cmd_key = 0xffeb  # Command key
+        space_key = 0x20  # Space key
+
+        # Press Command+Space
+        vnc.send_key_event(cmd_key, True)
+        vnc.send_key_event(space_key, True)
+
+        # Release Command+Space
+        vnc.send_key_event(space_key, False)
+        vnc.send_key_event(cmd_key, False)
+
+        # Small delay to let Spotlight open
+        time.sleep(0.5)
+
+        # Type the application name
+        vnc.send_text(identifier)
+
+        # Small delay to let Spotlight find the app
+        time.sleep(0.5)
+
+        # Press Enter to launch
+        enter_key = 0xff0d
+        vnc.send_key_event(enter_key, True)
+        vnc.send_key_event(enter_key, False)
+
+        end_time = time.time()
+        processing_time = round(end_time - start_time, 3)
+
+        return [types.TextContent(
+            type="text",
+            text=f"Launched application: {identifier}\nProcessing time: {processing_time}s"
+        )]
+
+    finally:
+        # Close VNC connection
+        vnc.close()
+
+
+def handle_remote_macos_mouse_drag(arguments: dict[str, Any]) -> list[types.TextContent | types.ImageContent | types.EmbeddedResource]:
+    """Perform a mouse drag operation on a remote MacOs machine."""
+    # Use environment variables
+    host = MACOS_HOST
+    port = MACOS_PORT
+    password = MACOS_PASSWORD
+    username = MACOS_USERNAME
+    encryption = VNC_ENCRYPTION
+
+    # Get required parameters from arguments
+    start_x = arguments.get("start_x")
+    start_y = arguments.get("start_y")
+    end_x = arguments.get("end_x")
+    end_y = arguments.get("end_y")
+    source_width = int(arguments.get("source_width", 1366))
+    source_height = int(arguments.get("source_height", 768))
+    button = int(arguments.get("button", 1))
+    steps = int(arguments.get("steps", 10))
+    delay_ms = int(arguments.get("delay_ms", 10))
+
+    # Validate required parameters
+    if any(x is None for x in [start_x, start_y, end_x, end_y]):
+        raise ValueError("start_x, start_y, end_x, and end_y coordinates are required")
+
+    # Ensure source dimensions are positive
+    if source_width <= 0 or source_height <= 0:
+        raise ValueError("Source dimensions must be positive values")
+
+    # Initialize VNC client
+    vnc = VNCClient(host=host, port=port, password=password, username=username, encryption=encryption)
+
+    # Connect to remote MacOs machine
+    success, error_message = vnc.connect()
+    if not success:
+        error_msg = f"Failed to connect to remote MacOs machine at {host}:{port}. {error_message}"
+        return [types.TextContent(type="text", text=error_msg)]
+
+    try:
+        # Get target screen dimensions
+        target_width = vnc.width
+        target_height = vnc.height
+
+        # Scale coordinates
+        scaled_start_x = int((start_x / source_width) * target_width)
+        scaled_start_y = int((start_y / source_height) * target_height)
+        scaled_end_x = int((end_x / source_width) * target_width)
+        scaled_end_y = int((end_y / source_height) * target_height)
+
+        # Ensure coordinates are within the screen bounds
+        scaled_start_x = max(0, min(scaled_start_x, target_width - 1))
+        scaled_start_y = max(0, min(scaled_start_y, target_height - 1))
+        scaled_end_x = max(0, min(scaled_end_x, target_width - 1))
+        scaled_end_y = max(0, min(scaled_end_y, target_height - 1))
+
+        # Calculate step sizes
+        dx = (scaled_end_x - scaled_start_x) / steps
+        dy = (scaled_end_y - scaled_start_y) / steps
+
+        # Move to start position
+        if not vnc.send_pointer_event(scaled_start_x, scaled_start_y, 0):
+            return [types.TextContent(type="text", text="Failed to move to start position")]
+
+        # Press button
+        button_mask = 1 << (button - 1)
+        if not vnc.send_pointer_event(scaled_start_x, scaled_start_y, button_mask):
+            return [types.TextContent(type="text", text="Failed to press mouse button")]
+
+        # Perform drag
+        for step in range(1, steps + 1):
+            current_x = int(scaled_start_x + dx * step)
+            current_y = int(scaled_start_y + dy * step)
+            if not vnc.send_pointer_event(current_x, current_y, button_mask):
+                return [types.TextContent(type="text", text=f"Failed during drag at step {step}")]
+            time.sleep(delay_ms / 1000.0)  # Convert ms to seconds
+
+        # Release button at final position
+        if not vnc.send_pointer_event(scaled_end_x, scaled_end_y, 0):
+            return [types.TextContent(type="text", text="Failed to release mouse button")]
+
+        # Prepare the response with useful details
+        scale_factors = {
+            "x": target_width / source_width,
+            "y": target_height / source_height
+        }
+
+        return [types.TextContent(
+            type="text",
+            text=f"""Mouse drag (button {button}) completed:
+From source ({start_x}, {start_y}) to ({end_x}, {end_y})
+From target ({scaled_start_x}, {scaled_start_y}) to ({scaled_end_x}, {scaled_end_y})
+Source dimensions: {source_width}x{source_height}
+Target dimensions: {target_width}x{target_height}
+Scale factors: {scale_factors['x']:.4f}x, {scale_factors['y']:.4f}y
+Steps: {steps}
+Delay: {delay_ms}ms"""
+        )]
+
     finally:
         # Close VNC connection
         vnc.close()

src/mcp_remote_macos_use/server.py

@@ -36,7 +36,9 @@
     handle_remote_macos_send_keys,
     handle_remote_macos_mouse_move,
     handle_remote_macos_mouse_click,
-    handle_remote_macos_mouse_double_click
+    handle_remote_macos_mouse_double_click,
+    handle_remote_macos_open_application,
+    handle_remote_macos_mouse_drag
 )
 
 # Configure logging
@@ -82,12 +84,12 @@
 async def main():
     """Run the Remote MacOS MCP server."""
     logger.info("Remote MacOS computer use server starting")
-    
+
     # Initialize LiveKit handler if environment variables are set
     livekit_handler = None
     if all([LIVEKIT_URL, LIVEKIT_API_KEY, LIVEKIT_API_SECRET]):
         livekit_handler = LiveKitHandler()
-        
+
         # Generate access token for the room
         token = api.AccessToken() \
             .with_identity("remote-macos-bot") \
@@ -96,7 +98,7 @@ async def main():
                 room_join=True,
                 room="remote-macos-room",
             )).to_jwt()
-        
+
         # Start LiveKit connection
         success = await livekit_handler.start("remote-macos-room", token)
         if success:
@@ -113,7 +115,7 @@ async def main():
     if not MACOS_PASSWORD:
         logger.error("MACOS_PASSWORD environment variable is required but not set")
         raise ValueError("MACOS_PASSWORD environment variable is required but not set")
-    
+
     server = Server("remote-macos-client")
 
     @server.list_resources()
@@ -147,8 +149,8 @@ async def handle_list_tools() -> list[types.Tool]:
                         "source_width": {"type": "integer", "description": "Width of the reference screen for coordinate scaling", "default": 1366},
                         "source_height": {"type": "integer", "description": "Height of the reference screen for coordinate scaling", "default": 768},
                         "direction": {
-                            "type": "string", 
-                            "description": "Scroll direction", 
+                            "type": "string",
+                            "description": "Scroll direction",
                             "enum": ["up", "down"],
                             "default": "down"
                         }
@@ -213,6 +215,39 @@ async def handle_list_tools() -> list[types.Tool]:
                     "required": ["x", "y"]
                 },
             ),
+            types.Tool(
+                name="remote_macos_open_application",
+                description="Opens/activates an application and returns its PID for further interactions.",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "identifier": {
+                            "type": "string",
+                            "description": "REQUIRED. App name, path, or bundle ID."
+                        }
+                    },
+                    "required": ["identifier"]
+                },
+            ),
+            types.Tool(
+                name="remote_macos_mouse_drag",
+                description="Perform a mouse drag operation from start point to end point on a remote MacOs machine, with automatic coordinate scaling. Uses environment variables for connection details.",
+                inputSchema={
+                    "type": "object",
+                    "properties": {
+                        "start_x": {"type": "integer", "description": "Starting X coordinate (in source dimensions)"},
+                        "start_y": {"type": "integer", "description": "Starting Y coordinate (in source dimensions)"},
+                        "end_x": {"type": "integer", "description": "Ending X coordinate (in source dimensions)"},
+                        "end_y": {"type": "integer", "description": "Ending Y coordinate (in source dimensions)"},
+                        "source_width": {"type": "integer", "description": "Width of the reference screen for coordinate scaling", "default": 1366},
+                        "source_height": {"type": "integer", "description": "Height of the reference screen for coordinate scaling", "default": 768},
+                        "button": {"type": "integer", "description": "Mouse button (1=left, 2=middle, 3=right)", "default": 1},
+                        "steps": {"type": "integer", "description": "Number of intermediate points for smooth dragging", "default": 10},
+                        "delay_ms": {"type": "integer", "description": "Delay between steps in milliseconds", "default": 10}
+                    },
+                    "required": ["start_x", "start_y", "end_x", "end_y"]
+                },
+            ),
         ]
 
     @server.call_tool()
@@ -223,25 +258,31 @@ async def handle_call_tool(
         try:
             if not arguments:
                 arguments = {}
-            
+
             if name == "remote_macos_get_screen":
                 return await handle_remote_macos_get_screen(arguments)
-                
+
             elif name == "remote_macos_mouse_scroll":
-                return handle_remote_macos_mouse_scroll(arguments)            
-            
+                return handle_remote_macos_mouse_scroll(arguments)
+
             elif name == "remote_macos_send_keys":
                 return handle_remote_macos_send_keys(arguments)
-            
+
             elif name == "remote_macos_mouse_move":
                 return handle_remote_macos_mouse_move(arguments)
-                    
+
             elif name == "remote_macos_mouse_click":
                 return handle_remote_macos_mouse_click(arguments)
-                    
+
             elif name == "remote_macos_mouse_double_click":
                 return handle_remote_macos_mouse_double_click(arguments)
-                    
+
+            elif name == "remote_macos_open_application":
+                return handle_remote_macos_open_application(arguments)
+
+            elif name == "remote_macos_mouse_drag":
+                return handle_remote_macos_mouse_drag(arguments)
+
             else:
                 raise ValueError(f"Unknown tool: {name}")
 
@@ -271,7 +312,7 @@ async def handle_call_tool(
 if __name__ == "__main__":
     # Load environment variables from .env file if it exists
     load_dotenv()
-    
+
     try:
         # Run the server
         asyncio.run(main())
@@ -282,4 +323,4 @@ async def handle_call_tool(
     except Exception as e:
         logger.error(f"Unexpected error: {str(e)}", exc_info=True)
         print(f"ERROR: Unexpected error occurred: {str(e)}")
-        sys.exit(1) 
+        sys.exit(1)