Miniwob zoom (#351)

* Set device scale factor to 1.0 in extract_screenshot for consistent rendering. This prevens screenshot sized to be affected by environments such as "retina" on macbook

* add examples to the description

* zoom miniwob 150% by default (doesn't change the viewport, just has a better usage of the viewport)

* adjust version

* Update extract_screenshot to use actual window dimensions for device metrics

* Enhance zoom functionality in miniwob for better viewport usage and precision in visual agents

* Enhance zoom_page docstring with detailed explanation of CSS transform scaling benefits

* Adjust zoom factor in miniwob to improve viewport usage and precision for visual agents

* adding type

* Remap the coordinate actions based on the zoom factor

* apply the zooming at the screenshot level

* changing the zoom method

Author: recursix

browsergym/assistantbench/requirements.txt

@@ -1,4 +1,4 @@
-browsergym-core==0.14.0
+browsergym-core==0.14.1
 datasets
 scipy
 numpy

browsergym/core/src/browsergym/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.14.0"
+__version__ = "0.14.1"
 
 import playwright.sync_api
 

browsergym/core/src/browsergym/core/action/functions.py

@@ -9,6 +9,7 @@
     call_fun,
     get_elem_by_bid,
     highlight_by_box,
+    map_coordinates,
     smooth_move_visual_cursor_to,
 )
 
@@ -277,6 +278,9 @@ def scroll(delta_x: float, delta_y: float):
         scroll(0, 200)
         scroll(-50.2, -100.5)
     """
+    delta_x, delta_y = map_coordinates(
+        page, delta_x, delta_y
+    )  # map coordinates to page coordinates
     page.mouse.wheel(delta_x, delta_y)
 
 
@@ -287,6 +291,8 @@ def scroll_at(x: int, y: int, dx: int, dy: int):
     Examples:
         scroll_at(50, 100, -50, -100)
     """
+    x, y = map_coordinates(page, x, y)  # map coordinates to page coordinates
+    dx, dy = map_coordinates(page, dx, dy)  # map coordinates to page coordinates
     page.mouse.move(x, y)  # position pointer
     page.mouse.wheel(dx, dy)
 
@@ -300,6 +306,7 @@ def mouse_move(x: float, y: float):
     Examples:
         mouse_move(65.2, 158.5)
     """
+    x, y = map_coordinates(page, x, y)  # map coordinates to page coordinates
     if demo_mode != "off":
         smooth_move_visual_cursor_to(page, x, y)
     page.mouse.move(x, y)
@@ -315,6 +322,7 @@ def mouse_up(x: float, y: float, button: Literal["left", "middle", "right"] = "l
         mouse_up(250, 120)
         mouse_up(47, 252, 'right')
     """
+    x, y = map_coordinates(page, x, y)  # map coordinates to page coordinates
     if demo_mode != "off":
         smooth_move_visual_cursor_to(page, x, y)
         highlight_by_box(page, {"x": x, "y": y, "width": 1, "height": 1})
@@ -332,6 +340,7 @@ def mouse_down(x: float, y: float, button: Literal["left", "middle", "right"] =
         mouse_down(140.2, 580.1)
         mouse_down(458, 254.5, 'middle')
     """
+    x, y = map_coordinates(page, x, y)  # map coordinates to page coordinates
     if demo_mode != "off":
         smooth_move_visual_cursor_to(page, x, y)
         highlight_by_box(page, {"x": x, "y": y, "width": 1, "height": 1})
@@ -349,6 +358,7 @@ def mouse_click(x: float, y: float, button: Literal["left", "middle", "right"] =
         mouse_click(887.2, 68)
         mouse_click(56, 712.56, 'right')
     """
+    x, y = map_coordinates(page, x, y)  # map coordinates to page coordinates
     if demo_mode != "off":
         smooth_move_visual_cursor_to(page, x, y)
         highlight_by_box(page, {"x": x, "y": y, "width": 1, "height": 1})
@@ -365,6 +375,7 @@ def mouse_dblclick(x: float, y: float, button: Literal["left", "middle", "right"
         mouse_dblclick(5, 236)
         mouse_dblclick(87.5, 354, 'right')
     """
+    x, y = map_coordinates(page, x, y)  # map coordinates to page coordinates
     if demo_mode != "off":
         smooth_move_visual_cursor_to(page, x, y)
         highlight_by_box(page, {"x": x, "y": y, "width": 1, "height": 1})
@@ -380,6 +391,9 @@ def mouse_drag_and_drop(from_x: float, from_y: float, to_x: float, to_y: float):
     Examples:
         mouse_drag_and_drop(10.7, 325, 235.6, 24.54)
     """
+    from_x, from_y = map_coordinates(page, from_x, from_y)  # map coordinates to page coordinates
+    to_x, to_y = map_coordinates(page, to_x, to_y)  # map coordinates to page coordinates
+
     if demo_mode != "off":
         x, y = from_x, from_y
         smooth_move_visual_cursor_to(page, x, y)
@@ -624,6 +638,7 @@ def mouse_upload_file(x: float, y: float, file: str | list[str]):
         mouse_upload_file(132.1, 547, "my_receipt.pdf")
         mouse_upload_file(328, 812, ["/home/bob/Documents/image.jpg", "/home/bob/Documents/file.zip"])
     """
+    x, y = map_coordinates(page, x, y)  # map coordinates to page coordinates
     if demo_mode != "off":
         smooth_move_visual_cursor_to(page, x, y)
         highlight_by_box(page, {"x": x, "y": y, "width": 1, "height": 1})

browsergym/core/src/browsergym/core/action/highlevel.py

@@ -515,7 +515,7 @@ def to_python_code(self, action):
         # return the constructed python code
         return python_code
 
-    def to_tool_description(self, api="openai") -> list[dict]:
+    def to_tool_description(self, api="openai", add_examples=True) -> list[dict]:
         """
         Translates actions to tool descriptions following the OpenAI API format.
 
@@ -535,16 +535,16 @@ def to_tool_description(self, api="openai") -> list[dict]:
             for param_name, param in signature.parameters.items():
                 param_type = "string"  # Default to string if type is not specified
                 if param.annotation != inspect.Parameter.empty:
-                    if param.annotation is str:
-                        param_type = "string"
-                    elif param.annotation is float or param.annotation is int:
-                        param_type = "number"
-                    elif param.annotation is bool:
-                        param_type = "boolean"
-                    elif param.annotation is dict:
-                        param_type = "object"
-                    elif param.annotation is list:
-                        param_type = "array"
+
+                    type_map = {
+                        str: "string",
+                        float: "number",
+                        int: "number",
+                        bool: "boolean",
+                        dict: "object",
+                        list: "array",
+                    }
+                    param_type = type_map.get(param.annotation, "string")
 
                 parameters["properties"][param_name] = {
                     "type": param_type,
@@ -554,9 +554,15 @@ def to_tool_description(self, api="openai") -> list[dict]:
                     parameters["required"].append(param_name)
 
             # Construct the tool descriptor
+            description = action.description
+            if add_examples and action.examples:
+                description += "\n\nExamples:\n"
+                for example in action.examples:
+                    description += f"- {example}\n"
+
             tool = {
                 "name": tool_name,
-                "description": action.description,
+                "description": description,
                 schema: parameters,
             }
             if api == "openai":

browsergym/core/src/browsergym/core/action/utils.py

@@ -286,3 +286,32 @@ def call_fun(fun: callable, retry_with_force: bool):
             fun(force=True)
         else:
             raise e
+
+
+def map_coordinates(page, x, y):
+    """
+    Maps screenshot coordinates back to page coordinates based on bgym_scale_factor.
+
+    When bgym_scale_factor > 1.0, the screenshot is captured at higher resolution,
+    so VLM coordinates need to be scaled down to match the actual page coordinates.
+
+    Args:
+        page: Playwright page object (should have _bgym_scale_factor attribute)
+        x: X coordinate from VLM (based on scaled screenshot)
+        y: Y coordinate from VLM (based on scaled screenshot)
+
+    Returns:
+        tuple: (mapped_x, mapped_y) coordinates for use with Playwright
+
+    Examples:
+        # If bgym_scale_factor = 1.5 and VLM returns coordinates (300, 450)
+        # from the scaled screenshot, map them back to page coordinates:
+        page_x, page_y = map_coordinates(page, 300, 450)  # Returns (200.0, 300.0)
+    """
+    scale_factor = getattr(page, "_bgym_scale_factor", 1.0)
+
+    # Scale down coordinates to match actual page coordinate system
+    mapped_x = x / scale_factor
+    mapped_y = y / scale_factor
+
+    return mapped_x, mapped_y

browsergym/core/src/browsergym/core/observation.py

@@ -115,6 +115,7 @@ def _post_extract(page: playwright.sync_api.Page):
 def extract_screenshot(page: playwright.sync_api.Page):
     """
     Extracts the screenshot image of a Playwright page using Chrome DevTools Protocol.
+    Uses the bgym_scale_factor to capture higher resolution screenshots for better VLM understanding.
 
     Args:
         page: the playwright page of which to extract the screenshot.
@@ -124,13 +125,45 @@ def extract_screenshot(page: playwright.sync_api.Page):
 
     """
 
+    scale_factor = getattr(page, "_bgym_scale_factor", 1.0)
     cdp = page.context.new_cdp_session(page)
+    viewport = page.viewport_size
+
+    if viewport is None:
+        dimensions = page.evaluate(
+            """() => ({
+            width: window.innerWidth,
+            height: window.innerHeight,
+            devicePixelRatio: window.devicePixelRatio
+        })"""
+        )
+    else:
+        dimensions = {
+            "width": viewport["width"],
+            "height": viewport["height"],
+            "devicePixelRatio": 1.0,  # Override system DPR
+        }
+
+    # Apply scale factor to device metrics for higher resolution capture
+    cdp.send(
+        "Emulation.setDeviceMetricsOverride",
+        {
+            "width": dimensions["width"],
+            "height": dimensions["height"],
+            "deviceScaleFactor": scale_factor,  # Use bgym_scale_factor here
+            "mobile": False,
+        },
+    )
+
     cdp_answer = cdp.send(
         "Page.captureScreenshot",
         {
             "format": "png",
         },
     )
+
+    # Reset device metrics
+    cdp.send("Emulation.clearDeviceMetricsOverride")
     cdp.detach()
 
     # bytes of a png file

browsergym/experiments/requirements.txt

@@ -1,3 +1,3 @@
-browsergym-core==0.14.0
+browsergym-core==0.14.1
 tiktoken>=0.4
 dataclasses-json

browsergym/experiments/src/browsergym/experiments/loop.py

@@ -17,6 +17,7 @@
 from pathlib import Path
 from typing import Optional
 
+from browsergym.core.env import BrowserEnv
 import gymnasium as gym
 import numpy as np
 from dataclasses_json import DataClassJsonMixin
@@ -49,7 +50,7 @@ class EnvArgs(DataClassJsonMixin):
 
     def make_env(
         self, action_mapping, exp_dir, exp_task_kwargs: dict = {}, use_raw_page_output=False
-    ):
+    ) -> BrowserEnv:
         """
         Instantiates the BrowserGym environment corresponding to the arguments (with some tweaks).
 

browsergym/miniwob/pyproject.toml

@@ -9,6 +9,7 @@ authors = [
     {name = "Rim Assouel"},
     {name = "Maxime Gasse"},
     {name = "Tom Marty"},
+    {name = "Alexandre Lacoste"},
 ]
 readme = "README.md"
 requires-python = ">3.7"

browsergym/miniwob/requirements.txt

@@ -1 +1 @@
-browsergym-core==0.14.0
+browsergym-core==0.14.1