Improve OmniParser integration with strict validation

- Add allow_no_parser flag to make it explicit when running without OmniParser
- Fail by default if OmniParser server is not available
- Update README with clear instructions for OmniParser configuration
- Add TODO for future Anthropic ComputerUse integration

🤖 Generated with [Claude Code](https://claude.ai/code)

Co-Authored-By: Claude <[email protected]>

Author: abrichr

omnimcp/README.md

@@ -69,11 +69,36 @@ omnimcp server
 # Run in debug mode to visualize screen elements
 omnimcp debug
 
+# Connect to a remote OmniParser server
+omnimcp cli --server-url=https://your-omniparser-server.example.com
+
+# Allow running even if OmniParser isn't available (limited functionality)
+omnimcp cli --allow-no-parser
+
 # With additional options
 omnimcp cli --use-normalized-coordinates
 omnimcp debug --debug-dir=/path/to/debug/folder
 ```
 
+### OmniParser Configuration
+
+OmniMCP requires access to an OmniParser server for analyzing screenshots:
+
+1. **Use a Remote OmniParser Server** (Recommended)
+   ```bash
+   omnimcp cli --server-url=https://your-omniparser-server.example.com
+   ```
+
+2. **Use the Default Local Server**
+   - OmniMCP will try to connect to `http://localhost:8000` by default
+   - This requires running an OmniParser server locally
+
+By default, OmniMCP will fail if it can't connect to an OmniParser server. Use the `--allow-no-parser` flag to run with limited functionality when no parser is available.
+
+### TODO: Anthropic ComputerUse Integration
+
+Future versions may integrate with Anthropic's ComputerUse system, which provides an official way for Claude to interact with computers through a virtualized desktop environment.
+
 ## Features
 
 - Visual UI analysis with OmniParser

openadapt/omnimcp.py

@@ -302,19 +302,22 @@ def __init__(
         self,
         server_url: Optional[str] = None,
         claude_api_key: Optional[str] = None,
-        use_normalized_coordinates: bool = False
+        use_normalized_coordinates: bool = False,
+        allow_no_parser: bool = False
     ):
         """Initialize OmniMCP.
         
         Args:
             server_url: URL of OmniParser server
             claude_api_key: API key for Claude (overrides config)
             use_normalized_coordinates: If True, use normalized (0-1) coordinates
+            allow_no_parser: If True, continue even if OmniParser is not available
         """
         self.omniparser = OmniParserProvider(server_url)
         self.visual_state = VisualState()
         self.claude_api_key = claude_api_key or config.ANTHROPIC_API_KEY
         self.use_normalized_coordinates = use_normalized_coordinates
+        self.allow_no_parser = allow_no_parser
 
         # Initialize controllers for keyboard and mouse
         self.keyboard_controller = keyboard.Controller()
@@ -329,6 +332,12 @@ def __init__(
         if not self.omniparser.is_available():
             logger.info("OmniParser not available, attempting to deploy...")
             self.omniparser.deploy()
+            
+            # Check again after deployment attempt
+            if not self.omniparser.is_available() and not allow_no_parser:
+                raise RuntimeError(
+                    "OmniParser server is not available. Please ensure it's running or use --allow-no-parser flag."
+                )
 
     def update_visual_state(self) -> VisualState:
         """Take screenshot and update visual state using OmniParser.
@@ -339,16 +348,31 @@ def update_visual_state(self) -> VisualState:
         # Take screenshot
         screenshot = utils.take_screenshot()
 
-        # Convert to bytes
-        img_byte_arr = io.BytesIO()
-        screenshot.save(img_byte_arr, format='PNG')
-        img_bytes = img_byte_arr.getvalue()
-        
-        # Parse with OmniParser
-        result = self.omniparser.parse_screenshot(img_bytes)
+        # Update the screenshot in visual state regardless of parser availability
+        self.visual_state.screenshot = screenshot
+        self.visual_state.timestamp = time.time()
 
-        # Update visual state
-        self.visual_state.update_from_omniparser(result, screenshot)
+        # If OmniParser is available, use it to analyze the screenshot
+        if self.omniparser.is_available():
+            # Convert to bytes
+            img_byte_arr = io.BytesIO()
+            screenshot.save(img_byte_arr, format='PNG')
+            img_bytes = img_byte_arr.getvalue()
+            
+            # Parse with OmniParser
+            result = self.omniparser.parse_screenshot(img_bytes)
+            
+            # Update visual state
+            self.visual_state.update_from_omniparser(result, screenshot)
+        elif not self.allow_no_parser:
+            # If parser not available and not allowed to continue without it, raise error
+            raise RuntimeError(
+                "OmniParser server is not available. Cannot update visual state."
+            )
+        else:
+            # If parser not available but allowed to continue, log warning
+            logger.warning("OmniParser not available. Visual state will have no UI elements.")
+            self.visual_state.elements = []
 
         return self.visual_state
 

openadapt/run_omnimcp.py

@@ -59,7 +59,8 @@ def cli(
         server_url=None,
         claude_api_key=None,
         use_normalized_coordinates=False,
-        debug_dir=None
+        debug_dir=None,
+        allow_no_parser=False
     ):
         """Run OmniMCP in CLI mode.
         
@@ -77,12 +78,14 @@ def cli(
             claude_api_key: Claude API key (if not provided, uses value from config.py)
             use_normalized_coordinates: Use normalized (0-1) coordinates instead of pixels
             debug_dir: Directory to save debug visualizations
+            allow_no_parser: If True, continue even if OmniParser is not available
         """
         # Create OmniMCP instance
         omnimcp = OmniMCP(
             server_url=server_url,
             claude_api_key=claude_api_key,  # Will use config.ANTHROPIC_API_KEY if None
-            use_normalized_coordinates=use_normalized_coordinates
+            use_normalized_coordinates=use_normalized_coordinates,
+            allow_no_parser=allow_no_parser
         )
 
         # Handle debug directory if specified
@@ -107,7 +110,8 @@ def server(
         server_url=None,
         claude_api_key=None,
         use_normalized_coordinates=False,
-        debug_dir=None
+        debug_dir=None,
+        allow_no_parser=False
     ):
         """Run OmniMCP as an MCP server.
         
@@ -127,12 +131,14 @@ def server(
             claude_api_key: Claude API key (if not provided, uses value from config.py)
             use_normalized_coordinates: Use normalized (0-1) coordinates instead of pixels
             debug_dir: Directory to save debug visualizations
+            allow_no_parser: If True, continue even if OmniParser is not available
         """
         # Create OmniMCP instance
         omnimcp = OmniMCP(
             server_url=server_url,
             claude_api_key=claude_api_key,  # Will use config.ANTHROPIC_API_KEY if None
-            use_normalized_coordinates=use_normalized_coordinates
+            use_normalized_coordinates=use_normalized_coordinates,
+            allow_no_parser=allow_no_parser
         )
 
         # Handle debug directory if specified
@@ -157,7 +163,8 @@ def debug(
         server_url=None,
         claude_api_key=None,
         use_normalized_coordinates=False,
-        debug_dir=None
+        debug_dir=None,
+        allow_no_parser=False
     ):
         """Run OmniMCP in debug mode.
         
@@ -174,12 +181,14 @@ def debug(
             claude_api_key: Claude API key (if not provided, uses value from config.py)
             use_normalized_coordinates: Use normalized (0-1) coordinates instead of pixels
             debug_dir: Directory to save debug visualizations
+            allow_no_parser: If True, continue even if OmniParser is not available
         """
         # Create OmniMCP instance
         omnimcp = OmniMCP(
             server_url=server_url,
             claude_api_key=claude_api_key,  # Will use config.ANTHROPIC_API_KEY if None
-            use_normalized_coordinates=use_normalized_coordinates
+            use_normalized_coordinates=use_normalized_coordinates,
+            allow_no_parser=allow_no_parser
         )
 
         # Create debug directory if not specified