Fix Problems for Online-Mind2Web (#207)

* Fix problems

* Fix debug string format

* fiix typos

* fix: remove self-defined _ensure_browser

Author: Genteki

environments/online_mind2web/Dockerfile

@@ -15,8 +15,8 @@ RUN uv pip install --python /opt/venv -e .
 # Create directories for logs and data
 RUN mkdir -p /app/logs /app/data
 
-ENV DISPLAY_WIDTH=1448
-ENV DISPLAY_HEIGHT=944
+ENV DISPLAY_WIDTH=1400
+ENV DISPLAY_HEIGHT=850
 
 ENV PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1

environments/online_mind2web/pyproject.toml

@@ -3,7 +3,7 @@ name = "hud-om2w"
 version = "0.1.0"
 description = "HUD Remote Browser Controller with MCP tools for cloud browser providers"
 requires-python = ">=3.11,<3.13"
-dependencies = [ "hud-python==0.4.61", "pyautogui", "playwright", "httpx", "typer", "google-api-python-client", "google-auth",]
+dependencies = [ "hud-python>=0.4.67", "anthropic>=0.74.0", "pyautogui", "playwright", "httpx", "typer", "google-api-python-client", "google-auth",]
 
 [build-system]
 requires = [ "hatchling",]

environments/online_mind2web/src/hud_controller/evaluate/webjudge.py

@@ -393,7 +393,7 @@ async def webjudge_eval(ctx: Context, task_description: dict | str, score_thresh
             main_score = int(scores[0]) if scores else 3
         except:
             main_score = 3  # Default score if parsing fails
-        logger.info("Score: ", main_score)
+        logger.info(f"Score: {main_score}")
 
         # Extract reasoning
         try:

environments/online_mind2web/src/hud_controller/providers/anchorbrowser.py

@@ -40,8 +40,8 @@ async def launch(self, **kwargs) -> str:
 
         Args:
             **kwargs: Launch options including:
-                - max_duration: Maximum session duration in seconds (default: 120)
-                - idle_timeout: Idle timeout in seconds (default: 30)
+                - max_duration: Maximum session duration in seconds (default: 300)
+                - idle_timeout: Idle timeout in seconds (default: 120)
                 - proxy: Proxy configuration dict with:
                     - type: "custom" or "anchor_residential"
                     - server: Proxy server address (for custom)
@@ -61,8 +61,8 @@ async def launch(self, **kwargs) -> str:
         request_data = {
             "session": {
                 "timeout": {
-                    "max_duration": kwargs.get("max_duration", 120),
-                    "idle_timeout": kwargs.get("idle_timeout", 30),
+                    "max_duration": kwargs.get("max_duration", 300),
+                    "idle_timeout": kwargs.get("idle_timeout", 120),
                 },
             },
             "browser": {
@@ -72,6 +72,19 @@ async def launch(self, **kwargs) -> str:
             },
         }
 
+        # Add viewport configuration
+        if "viewport" in kwargs:
+            request_data["browser"]["viewport"] = kwargs["viewport"]
+        else:
+            # Use environment variables or AnchorBrowser's recommended default (1440x900)
+            request_data["browser"]["viewport"] = {
+                "width": int(os.getenv("DISPLAY_WIDTH", "1440")),
+                "height": int(os.getenv("DISPLAY_HEIGHT", "900")),
+            }
+            logger.info(
+                f"Setting viewport to {request_data['browser']['viewport']['width']}x{request_data['browser']['viewport']['height']}"
+            )
+
         proxy_config = await get_proxy_config()
 
         # Default to residential proxy if nothing configured

environments/online_mind2web/src/hud_controller/server.py

@@ -26,7 +26,7 @@
     OpenAIComputerTool,
     HudComputerTool,
 )
-from hud.tools import PlaywrightTool
+from .tools import OnlineMind2Web_PlaywrightTool as PlaywrightTool
 
 # Import setup and evaluate hubs
 from .setup import setup as setup_hub
@@ -283,10 +283,14 @@ async def send_progress(progress: int, message: str):
 
         # Create and register computer tools with default dimensions
         mcp.add_tool(HudComputerTool(executor=browser_executor))
-        # mcp.add_tool(AnthropicComputerTool(executor=browser_executor))
-        # mcp.add_tool(OpenAIComputerTool(executor=browser_executor))
-        mcp.add_tool(AnthropicComputerToolWithRecord(executor=browser_executor))
-        mcp.add_tool(OpenAIComputerToolWithRecord(executor=browser_executor))
+        mcp.add_tool(
+            AnthropicComputerToolWithRecord(
+                executor=browser_executor, name="anthropic_computer_tool"
+            )
+        )
+        mcp.add_tool(
+            OpenAIComputerToolWithRecord(executor=browser_executor, name="openai_computer_tool")
+        )
 
         await send_progress(80, "Registered hud computer tools")
 

environments/online_mind2web/src/hud_controller/setup/navigate.py

@@ -9,7 +9,7 @@
 
 
 @setup.tool("navigate_to_url")
-async def navigate_to_url(ctx: Context, url: str, wait_for_load_state: str = "networkidle"):
+async def navigate_to_url(ctx: Context, url: str, wait_for_load_state: str = "load"):
     """Navigate browser to a specific URL.
 
     Args:

environments/online_mind2web/src/hud_controller/tools/__init__.py

@@ -3,9 +3,11 @@
 from .executor import BrowserExecutor
 from .anthropic import AnthropicComputerToolWithRecord
 from .openai import OpenAIComputerToolWithRecord
+from .playwright import OnlineMind2Web_PlaywrightTool
 
 __all__ = [
     "BrowserExecutor",
     "AnthropicComputerToolWithRecord",
     "OpenAIComputerToolWithRecord",
+    "OnlineMind2Web_PlaywrightTool",
 ]

environments/online_mind2web/src/hud_controller/tools/anthropic.py

@@ -118,6 +118,15 @@ async def _on_recorded_action(
         except Exception as e:
             logger.warning(f"Failed to record action: {e}")
 
+    def _scale_coordinates(self, x: int | None, y: int | None) -> tuple[int | None, int | None]:
+        """Scale coordinates from target space to screen space."""
+        if x is not None and self.scale_x != 1.0:
+            x = round(x / self.scale_x)
+        if y is not None and self.scale_y != 1.0:
+            y = round(y / self.scale_y)
+
+        return x, y
+
     async def __call__(
         self,
         action: str = Field(..., description="The action to perform on the computer"),
@@ -171,6 +180,12 @@ async def __call__(
         }
         if action in screenshot_actions and action != "screenshot" and take_screenshot_on_click:
             await self._trigger_callbacks("on_screenshot_action")
+            logger.debug(
+                "Env display size %s x %s",
+                self.environment_width,
+                self.environment_height,
+            )
+
         recorded_actions = {
             "left_click",
             "click",

environments/online_mind2web/src/hud_controller/tools/executor.py

@@ -112,11 +112,16 @@ async def _ensure_page(self):
     async def screenshot(self) -> str | None:
         """Take a screenshot and return base64 encoded image."""
         try:
-            page = await self._ensure_page()
-            screenshot_bytes = await page.screenshot(full_page=False)
-            screenshot_b64 = base64.b64encode(screenshot_bytes).decode()
-            logger.debug("Browser screenshot captured")
-            return screenshot_b64
+            result = await self.playwright_tool.screenshot()
+            if result.base64_image:
+                logger.debug("Browser screenshot captured via playwright tool")
+                return result.base64_image
+            elif result.error:
+                logger.error(f"Screenshot failed: {result.error}")
+                return None
+            else:
+                logger.error("Screenshot returned no image or error")
+                return None
         except Exception as e:
             logger.error(f"Screenshot failed: {e}")
             return None