Some changes in preparation for agentlab's new ToolUseAgent and the new APIs (ServiceNow#340)

* added openai/anthropic compatible tool json descriptions

* added raw page as possible output to reduce overhead

* moving the benchmark section out of bgym (to agentlab)

* Update browsergym/core/src/browsergym/core/action/highlevel.py

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>

* adding benchmark stuff back in

* backtracking

* backtracking everything

* brought back the 'eval' bc we need it actually

* fixing broken imports

* version bump

* switching eval func to globals()

* Update loop.py

* Update browsergym/experiments/src/browsergym/experiments/loop.py

* Add scroll_at function and update references in highlevel.py; refactor browser launch args in env.py; clean up utils.py

* Replace 'scroll' with 'scroll_at' in ACTION_SUBSETS for clarity and consistency

* Refactor ACTION_SUBSETS to replace 'scroll_at' with 'scroll' for consistency; update browser launch args comments in env.py

* Update test_scroll to include 'bid' in action_set subsets for improved functionality; add main execution block for direct testing

---------

Co-authored-by: Copilot <175728472+Copilot@users.noreply.github.com>
Co-authored-by: recursix <alex.lacoste.shmu@gmail.com>

Author: 3 people

browsergym/assistantbench/requirements.txt

@@ -1,4 +1,4 @@
-browsergym-core==0.13.4
+browsergym-core==0.14.0
 datasets
 scipy
 numpy

browsergym/core/src/browsergym/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.13.4"
+__version__ = "0.14.0"
 
 import playwright.sync_api
 

browsergym/core/src/browsergym/core/action/base.py

@@ -1,4 +1,5 @@
 from abc import ABC, abstractmethod
+from typing import Any
 
 import playwright.sync_api
 
@@ -33,6 +34,15 @@ def to_python_code(self, action) -> str:
             Executable python code that performs the action in a browsergym environment.
         """
 
+    def to_tool_descriptor(self) -> list[Any]:
+        """
+        Converts the action set to a tool descriptor.
+
+        Returns:
+            A list of dictionaries describing the actions in the action set.
+        """
+        pass
+
 
 def execute_python_code(
     code: str,

browsergym/core/src/browsergym/core/action/functions.py

@@ -280,6 +280,17 @@ def scroll(delta_x: float, delta_y: float):
     page.mouse.wheel(delta_x, delta_y)
 
 
+def scroll_at(x: int, y: int, dx: int, dy: int):
+    """
+    Scroll horizontally and vertically. Amounts in pixels, positive for right or down scrolling, negative for left or up scrolling. Dispatches a wheel event.
+
+    Examples:
+        scroll_at(50, 100, -50, -100)
+    """
+    page.mouse.move(x, y)  # position pointer
+    page.mouse.wheel(dx, dy)
+
+
 # https://playwright.dev/python/docs/api/class-mouse#mouse-move
 def mouse_move(x: float, y: float):
     """

browsergym/core/src/browsergym/core/action/highlevel.py

@@ -33,6 +33,7 @@
     press,
     report_infeasible,
     scroll,
+    scroll_at,
     select_option,
     send_msg_to_user,
     tab_close,
@@ -61,7 +62,7 @@
         upload_file,
     ],
     "coord": [
-        scroll,
+        scroll_at,
         mouse_move,
         mouse_up,
         mouse_down,
@@ -514,6 +515,56 @@ def to_python_code(self, action):
         # return the constructed python code
         return python_code
 
+    def to_tool_description(self, api="openai") -> list[dict]:
+        """
+        Translates actions to tool descriptions following the OpenAI API format.
+
+        Returns:
+            A list of tool descriptors.
+        """
+        schema_keys = {
+            "openai": "parameters",
+            "anthropic": "input_schema",
+        }
+        schema = schema_keys.get(api, "parameters")
+        tools = []
+        for tool_name, action in self.action_set.items():
+            # Parse the signature to extract parameter names and types
+            parameters = {"type": "object", "properties": {}, "required": []}
+            signature = inspect.signature(globals()[tool_name])
+            for param_name, param in signature.parameters.items():
+                param_type = "string"  # Default to string if type is not specified
+                if param.annotation != inspect.Parameter.empty:
+                    if param.annotation is str:
+                        param_type = "string"
+                    elif param.annotation is float or param.annotation is int:
+                        param_type = "number"
+                    elif param.annotation is bool:
+                        param_type = "boolean"
+                    elif param.annotation is dict:
+                        param_type = "object"
+                    elif param.annotation is list:
+                        param_type = "array"
+
+                parameters["properties"][param_name] = {
+                    "type": param_type,
+                    # "description": f"Parameter {param_name} of type {param_type}",
+                }
+                if param.default == inspect.Parameter.empty:
+                    parameters["required"].append(param_name)
+
+            # Construct the tool descriptor
+            tool = {
+                "name": tool_name,
+                "description": action.description,
+                schema: parameters,
+            }
+            if api == "openai":
+                tool["type"] = "function"
+            tools.append(tool)
+
+        return tools
+
 
 # consistency checks
 assert "custom" not in ACTION_SUBSETS

browsergym/core/src/browsergym/core/env.py

@@ -25,7 +25,7 @@
     extract_merged_axtree,
     extract_screenshot,
 )
-from .spaces import AnyBox, AnyDict, Float, Unicode
+from .spaces import AnyBox, AnyDict, Anything, Float, Unicode
 from .task import AbstractBrowserTask
 
 logger = logging.getLogger(__name__)
@@ -76,6 +76,7 @@ def __init__(
         pw_context_kwargs: dict = {},
         # agent-related arguments
         action_mapping: Optional[callable] = HighLevelActionSet().to_python_code,
+        use_raw_page_output: bool = False,
     ):
         """
         Instantiate a ready to use BrowserEnv gym environment.
@@ -96,6 +97,7 @@ def __init__(
             pw_chromium_kwargs: extra parameters for the playwright Browser. Should only be used for debugging/testing.
             pw_context_kwargs: extra parameters for the playwright BrowserContext. Should only be used for debugging/testing.
             action_mapping: if set, the environment will use this function to map every received action to executable Python code.
+            use_raw_page_output: if set, the environment will use the raw page output instead of the default processing.
 
         """
         super().__init__()
@@ -115,6 +117,7 @@ def __init__(
         self.pw_chromium_kwargs = pw_chromium_kwargs
         self.pw_context_kwargs = pw_context_kwargs
         self.action_mapping = action_mapping
+        self.use_raw_page_output = use_raw_page_output
 
         # check argument values
         assert tags_to_mark in ("all", "standard_html")
@@ -132,42 +135,67 @@ def __init__(
         self.chat: Chat = None
 
         # observation space
-        self.observation_space = gym.spaces.Dict(
-            {
-                "chat_messages": gym.spaces.Sequence(
-                    gym.spaces.Dict(
-                        {
-                            "role": Unicode(),
-                            "timestamp": Float(),
-                            "message": Unicode(),
-                        }
-                    )
-                ),
-                "goal": Unicode(),
-                "goal_object": gym.spaces.Sequence(AnyDict()),
-                "open_pages_urls": gym.spaces.Sequence(Unicode()),
-                "open_pages_titles": gym.spaces.Sequence(Unicode()),
-                "active_page_index": gym.spaces.Box(
-                    low=0, high=255, dtype=int
-                ),  # TODO: change to an Integer (breaking change for users)
-                "url": Unicode(),
-                "screenshot": AnyBox(
-                    low=0,
-                    high=255,
-                    shape=(-1, -1, 3),
-                    dtype=np.uint8,
-                ),  # swapped axes (height, width, RGB)
-                "dom_object": AnyDict(),
-                "axtree_object": AnyDict(),
-                "extra_element_properties": AnyDict(),
-                "focused_element_bid": Unicode(),
-                "last_action": Unicode(),
-                "last_action_error": Unicode(),
-                "elapsed_time": gym.spaces.Box(
-                    low=0, high=np.inf, dtype=float
-                ),  # TODO: change to a Float (breaking change for users)
-            }
-        )
+        if use_raw_page_output:
+            self.observation_space = gym.spaces.Dict(
+                {
+                    "page": Anything(),
+                    "chat_messages": gym.spaces.Sequence(
+                        gym.spaces.Dict(
+                            {
+                                "role": Unicode(),
+                                "timestamp": Float(),
+                                "message": Unicode(),
+                            }
+                        )
+                    ),
+                    "goal": Unicode(),
+                    "goal_object": gym.spaces.Sequence(AnyDict()),
+                    "open_pages_urls": gym.spaces.Sequence(Unicode()),
+                    "open_pages_titles": gym.spaces.Sequence(Unicode()),
+                    "active_page_index": gym.spaces.Box(low=0, high=255, dtype=int),
+                    "url": Unicode(),
+                    "last_action": Unicode(),
+                    "last_action_error": Unicode(),
+                    "elapsed_time": gym.spaces.Box(low=0, high=np.inf, dtype=float),
+                }
+            )
+        else:
+            self.observation_space = gym.spaces.Dict(
+                {
+                    "chat_messages": gym.spaces.Sequence(
+                        gym.spaces.Dict(
+                            {
+                                "role": Unicode(),
+                                "timestamp": Float(),
+                                "message": Unicode(),
+                            }
+                        )
+                    ),
+                    "goal": Unicode(),
+                    "goal_object": gym.spaces.Sequence(AnyDict()),
+                    "open_pages_urls": gym.spaces.Sequence(Unicode()),
+                    "open_pages_titles": gym.spaces.Sequence(Unicode()),
+                    "active_page_index": gym.spaces.Box(
+                        low=0, high=255, dtype=int
+                    ),  # TODO: change to an Integer (breaking change for users)
+                    "url": Unicode(),
+                    "screenshot": AnyBox(
+                        low=0,
+                        high=255,
+                        shape=(-1, -1, 3),
+                        dtype=np.uint8,
+                    ),  # swapped axes (height, width, RGB)
+                    "dom_object": AnyDict(),
+                    "axtree_object": AnyDict(),
+                    "extra_element_properties": AnyDict(),
+                    "focused_element_bid": Unicode(),
+                    "last_action": Unicode(),
+                    "last_action_error": Unicode(),
+                    "elapsed_time": gym.spaces.Box(
+                        low=0, high=np.inf, dtype=float
+                    ),  # TODO: change to a Float (breaking change for users)
+                }
+            )
 
         # action space
         self.action_space = Unicode()
@@ -227,16 +255,24 @@ def override_property(task, env, property):
         pw: playwright.sync_api.Playwright = _get_global_playwright()
         # important: change playwright's test id attribute from "data-testid" to "bid"
         pw.selectors.set_test_id_attribute(BROWSERGYM_ID_ATTRIBUTE)
+        args = [
+            (
+                f"--window-size={viewport['width']},{viewport['height']}"
+                if self.resizeable_window
+                else None
+            ),
+            "--disable-features=OverlayScrollbars,ExtendedOverlayScrollbars",  # otherwise the screenshot doesn't see the scrollbars
+        ]
+        args = [arg for arg in args if arg is not None]  # Remove None values
 
         # create a new browser
         self.browser = pw.chromium.launch(
             headless=self.headless,
             slow_mo=slow_mo,
-            args=(
-                [f"--window-size={viewport['width']},{viewport['height']}"]
-                if self.resizeable_window
-                else None
-            ),
+            args=args,
+            ignore_default_args=[
+                "--hide-scrollbars"
+            ],  # otherwise the screenshot doesn't see the scrollbars
             # will raise an Exception if above args are overriden
             **self.pw_chromium_kwargs,
         )
@@ -566,7 +602,23 @@ def _active_page_check(self):
             raise RuntimeError(f"Unexpected: active page has been closed ({self.page}).")
 
     def _get_obs(self):
-
+        if self.use_raw_page_output:
+            obs = {
+                "page": self.page,
+                "chat_messages": tuple(copy.deepcopy(self.chat.messages)),
+                "goal": _try_to_extract_legacy_goal(self.goal_object),  # legacy goal, deprecated
+                "goal_object": tuple(
+                    copy.deepcopy(self.goal_object)
+                ),  # new goal format, list of messages openai style
+                "open_pages_urls": tuple(page.url for page in self.context.pages),
+                "open_pages_titles": tuple(page.title() for page in self.context.pages),
+                "active_page_index": np.asarray([self.context.pages.index(self.page)]),
+                "url": self.page.url,  # redundant with "open_pages_urls" and "active_page_index"
+                "last_action": self.last_action,
+                "last_action_error": self.last_action_error,
+                "elapsed_time": np.asarray([time.time() - self.start_time]),
+            }
+            return obs
         for retries_left in reversed(range(EXTRACT_OBS_MAX_TRIES)):
             try:
                 # pre-extraction, mark dom elements (set bid, set dynamic attributes like value and checked)

browsergym/experiments/requirements.txt

@@ -1,3 +1,3 @@
-browsergym-core==0.13.4
+browsergym-core==0.14.0
 tiktoken>=0.4
 dataclasses-json

browsergym/experiments/src/bgym/__init__.py

@@ -2,11 +2,7 @@
 from browsergym.core.action.highlevel import HighLevelActionSet
 from browsergym.core.action.python import PythonActionSet
 from browsergym.experiments.agent import Agent, AgentInfo
-from browsergym.experiments.benchmark import (
-    DEFAULT_BENCHMARKS,
-    Benchmark,
-    HighLevelActionSetArgs,
-)
+from browsergym.experiments.benchmark import DEFAULT_BENCHMARKS, Benchmark, HighLevelActionSetArgs
 from browsergym.experiments.loop import (
     AbstractAgentArgs,
     EnvArgs,

browsergym/experiments/src/browsergym/experiments/benchmark/configs.py

@@ -1,9 +1,6 @@
 import numpy as np
 
-from browsergym.experiments.benchmark.metadata.utils import (
-    task_list_from_metadata,
-    task_metadata,
-)
+from browsergym.experiments.benchmark.metadata.utils import task_list_from_metadata, task_metadata
 from browsergym.experiments.benchmark.utils import (
     make_env_args_list_from_fixed_seeds,
     make_env_args_list_from_repeat_tasks,

browsergym/experiments/src/browsergym/experiments/benchmark/utils.py

@@ -6,7 +6,6 @@
 from typing import Literal
 
 import numpy as np
-
 from browsergym.experiments.loop import SEED_MAX, EnvArgs
 
 logger = logging.getLogger(__name__)
@@ -46,7 +45,11 @@ def make_env_args_list_from_workarena_curriculum(
 
 
 def make_env_args_list_from_repeat_tasks(
-    task_list: list[str], max_steps: int, n_repeats: int, seeds_rng: np.random.RandomState
+    task_list: list[str],
+    max_steps: int,
+    n_repeats: int,
+    seeds_rng: np.random.RandomState,
+    viewport=None,
 ):
     """
     Generates a list of `len(task_list)` time `n_repeats` environments arguments, using randomly generated seeds.
@@ -62,7 +65,7 @@ def make_env_args_list_from_repeat_tasks(
                     headless=True,
                     record_video=False,
                     wait_for_user_message=False,
-                    viewport=None,
+                    viewport=viewport,
                     slow_mo=None,
                     storage_state=None,
                     task_kwargs=None,