Merge branch 'main' of github.com:ServiceNow/BrowserGym

Author: optimass

README.md

@@ -4,20 +4,20 @@ This package provides `browsergym`, a gym environment for web task automation in
 
 ## Setup
 
-1. Install the package
+To install browsergym, you can either install one of the `browsergym-miniwob`, `browsergym-webarena` and `browsergym-workarena` packages, or you can simply install `browsergym` which includes all of these by default.
 ```sh
 pip install browsergym
 ```
 
-2. Install playwright browsers
+Then, a required step is to setup playwright by running
 ```sh
 playwright install
 ```
 
-3. For each benchmark, specific setup instructions must be followed
+Finally, each benchmark comes with its its own specific setup that requires to follow additional steps.
  - for miniwob, see [miniwob/README.md](miniwob/README.md)
  - for webarena, see [webarena/README.md](webarena/README.md)
- - for workarena, see the [WorkArena](https://github.com/ServiceNow/WorkArena/README.md) repository
+ - for workarena, see the [WorkArena README](https://github.com/ServiceNow/WorkArena/README.md)
 
 4. (optional) if you want to run the UI assistant to create demos
 
@@ -33,10 +33,10 @@ playwright install
 
 ### Open-ended task example
 
-Boilerplate code to run an agent on the openended task:
+Boilerplate code to run an agent on an interactive, openended task:
 ```python
 import gymnasium as gym
-import browsergym.core  # register openended taskj as a gym environment
+import browsergym.core  # register the openended task as a gym environment
 
 env = gym.make(
     "browsergym/openended", start_url="https://www.google.com/", wait_for_user_message=True
@@ -63,6 +63,12 @@ while not done:
     obs, reward, terminated, truncated, info = env.step(action)
 ```
 
+List of all the available MiniWoB++ environments
+```python
+env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/miniwob")]
+print("\n".join(env_ids))
+```
+
 ### WebArena task example
 
 Boilerplate code to run an agent on a webarena task:
@@ -78,6 +84,12 @@ while not done:
     obs, reward, terminated, truncated, info = env.step(action)
 ```
 
+List of all the available WebArena environments
+```python
+env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/webarena")]
+print("\n".join(env_ids))
+```
+
 ### WorkArena task example
 
 Boilerplate code to run an agent on a workarena task:
@@ -93,5 +105,8 @@ while not done:
     obs, reward, terminated, truncated, info = env.step(action)
 ```
 
-## UI assistant
-
+List of all the available WorkArena environments
+```python
+env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/workarena")]
+print("\n".join(env_ids))
+```

core/README.md

@@ -8,17 +8,3 @@ This package provides `browsergym.core`, which provides the core functionalities
 ```sh
 pip install browsergym-core
 ```
-
-2. Install playwright browsers
-```sh
-playwright install
-```
-
-## Usage
-
-```python
-import gymnasium as gym
-import browsergym.core  # register gym environments
-
-env = gym.make("browsergym/openended")
-```

core/src/browsergym/core/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.0.1a10"
+__version__ = "0.1.0rc0"
 
 import playwright.sync_api
 

core/src/browsergym/core/chat_files/chatbox.html

@@ -41,7 +41,7 @@
 
         .chat-debug {
             padding: 10px;
-            max-height: 20%;
+            max-height: 30%;
             overflow-y: auto;
             display: flex;
             flex-direction: column;

core/src/browsergym/core/env.py

@@ -147,7 +147,9 @@ def reset(self, seed=None, *args, **kwargs):
         )
         # create the chat at the same time to make sure videos are synced
         self.chat = Chat(
-            headless=self.playwright_kwargs["headless"], record_video_dir=self.record_video_dir
+            headless=self.playwright_kwargs["headless"],
+            chat_size=(500, max(self.viewport["height"], 800)),
+            record_video_dir=self.record_video_dir,
         )
         t_after = time.time()
         recording_start_time = (t_before + t_after) / 2  # recording start time
@@ -215,6 +217,9 @@ def reset(self, seed=None, *args, **kwargs):
         self.last_action = ""
         self.last_action_error = ""
 
+        # if asked, wait for user message
+        self._wait_for_user_message()
+
         # extract obs and info from environment
         obs = self._get_obs()
 
@@ -245,21 +250,24 @@ def step(self, action: str) -> tuple:
         time.sleep(0.5)  # wait for JS events to be fired (half a second)
         self.context.cookies()  # trigger all waiting Playwright callbacks on the stack (hack, see https://playwright.dev/java/docs/multithreading)
 
+        # wait for the network to idle before extracting the observation, reward etc.
+        self._wait_dom_loaded()
+
         # after the action is executed, the active page might have changed
         # perform a safety check
         self._active_page_check()
 
-        # wait for the network to idle before extracting the observation, reward etc.
-        self._wait_dom_loaded()
+        # if asked, wait for user message
+        self._wait_for_user_message()
 
-        # reward, done, user_message, info are task-specific
+        # extract reward, done, user_message, info (task-specific)
         reward, done, user_message, info = self.task.validate(self.page, self.chat.messages)
 
         # add any user message sent by the task to the chat
         if user_message:
             self.chat.add_message(role="user", msg=user_message)
 
-        # extract obs, reward etc.
+        # extract observation (generic)
         obs = self._get_obs()
 
         # new step API wants a 5-tuple (gymnasium)
@@ -268,6 +276,12 @@ def step(self, action: str) -> tuple:
 
         return obs, reward, terminated, truncated, info
 
+    def _wait_for_user_message(self):
+        # if last message is from the assistant, wait for a user message to continue
+        # TODO: be smarter about when to wait for a user message (different action from the assistant?)
+        if self.chat.messages[-1]["role"] == "assistant" and self.wait_for_user_message:
+            self.chat.wait_for_user_message()
+
     def _wait_dom_loaded(self):
         for page in self.context.pages:
             try:
@@ -322,11 +336,6 @@ def _active_page_check(self):
 
     def _get_obs(self):
 
-        # if last message is from the assistant, wait for a user message to continue
-        # TODO: be smarter about when to wait for a user message (different action from the assistant?)
-        if self.chat.messages[-1]["role"] == "assistant" and self.wait_for_user_message:
-            self.chat.wait_for_user_message()
-
         for retries_left in reversed(range(EXTRACT_OBS_MAX_TRIES)):
             try:
                 # pre-extraction, mark dom elements (set bid, set dynamic attributes like value and checked)

miniwob/README.md

@@ -21,18 +21,3 @@ export MINIWOB_URL="file://<PATH_TO_MINIWOB_CLONED_REPO>/miniwob/html/miniwob/"
 ```
 
 Alternatively, one can [setup a simple HTTP server](https://miniwob.farama.org/content/viewing/) and use a proper URL.
-
-## Usage
-
-```python
-import gymnasium as gym
-import browsergym.miniwob  # register gym environments
-
-env = gym.make("browsergym/miniwob.choose-list")
-```
-
-List of all the available MiniWoB++ environments
-```python
-env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/miniwob")]
-print("\n".join(env_ids))
-```

miniwob/requirements.txt

@@ -1 +1 @@
-browsergym-core==0.0.1a10
+browsergym-core==0.1.0rc0

miniwob/src/browsergym/miniwob/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.0.1a10"
+__version__ = "0.1.0rc0"
 
 from browsergym.core.registration import register_task
 

pyproject.toml

@@ -25,12 +25,12 @@ classifiers = [
     "Topic :: Scientific/Engineering :: Artificial Intelligence",
     "License :: OSI Approved :: Apache Software License",
 ]
-version="0.0.1a10"
+version="0.1.0rc0"
 dependencies = [
-    "browsergym-core==0.0.1a10",
-    "browsergym-miniwob==0.0.1a10",
-    "browsergym-webarena==0.0.1a10",
-    "browsergym-workarena==0.0.1a10",
+    "browsergym-core==0.1.0rc0",
+    "browsergym-miniwob==0.1.0rc0",
+    "browsergym-webarena==0.1.0rc0",
+    "browsergym-workarena==0.1.0rc0",
 ]
 
 [tool.setuptools]

webarena/README.md

@@ -31,18 +31,3 @@ export WIKIPEDIA="$BASE_URL:8888/wikipedia_en_all_maxi_2022-05/A/User:The_other_
 export MAP="$BASE_URL:3000"
 export HOMEPAGE="$BASE_URL:4399"
 ```
-
-## Usage
-
-```python
-import gymnasium as gym
-import browsergym.webarena  # register gym environments
-
-env = gym.make("browsergym/webarena.310")
-```
-
-List of all the available WebArena environments
-```python
-env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/webarena")]
-print("\n".join(env_ids))
-```