playwright backend

Author: ollmer

experiments/run_miniwob.py

@@ -8,6 +8,7 @@
 from agentlab.agents.generic_agent.generic_agent import GenericAgentArgs
 from agentlab.agents.tapeagent.agent import TapeAgentArgs, load_config
 from agentlab.backends.browser.mcp_playwright import MCPPlaywright
+from agentlab.backends.browser.playwright import PlaywrightSyncBackend
 from agentlab.benchmarks.miniwob import MiniWobBenchmark
 from agentlab.experiments.study import make_study
 from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT
@@ -20,13 +21,16 @@
 if __name__ == "__main__":
     config = load_config("miniwob")
 
-    # benchmark = DEFAULT_BENCHMARKS["miniwob"]()
+    # benchmark = DEFAULT_BENCHMARKS["miniwob"](n_repeats=1)
+    benchmark = MiniWobBenchmark(backend=PlaywrightSyncBackend())
+
     agent_args = GenericAgentArgs(
         chat_model_args=CHAT_MODEL_ARGS_DICT["anthropic/claude-sonnet-4-20250514"],
         flags=FLAGS_GPT_4o,
     )
-
-    benchmark = MiniWobBenchmark(backend=MCPPlaywright())
+    # agent_args.flags.obs.use_ax_tree = False
+    # agent_args.flags.obs.use_html = True
+    # agent_args.flags.obs.use_focused_element = False
     # agent_args =TapeAgentArgs(agent_name=config.name, config=config)
 
 
@@ -38,7 +42,7 @@
         logging_level_stdout=logging.INFO,
     )
     if os.environ.get("AGENTLAB_DEBUG"):
-        study.exp_args_list = study.exp_args_list[1:2]
+        study.exp_args_list = study.exp_args_list[23:24]
         study.run(n_jobs=1, n_relaunch=1, parallel_backend="sequential")
     else:
         study.run(n_jobs=config.n_jobs, n_relaunch=1, parallel_backend=config.parallel_backend)

src/agentlab/backends/browser/playwright.py

@@ -1,53 +1,111 @@
-from playwright.sync_api import sync_playwright
+import logging
+from io import BytesIO
+from typing import Any, Callable
 
-from agentlab.backends.browser.base import BrowserBackend, ToolCallAction
+from PIL import Image
+from playwright.sync_api import Page, sync_playwright
+
+from agentlab.backends.browser.base import BrowserBackend, ToolCallAction, ToolSpec
+
+logger = logging.getLogger(__name__)
 
 
 class PlaywrightSyncBackend(BrowserBackend):
-    def __init__(self):
-        self.actions = {
-            "browser_press_key": lambda key: self.page.keyboard.press(key),
-            "browser_type": lambda text: self.page.type(text),
-            "browser_click": lambda selector: self.page.click(selector),
-            "browser_drag": lambda from_selector, to_selector: self.drag_and_drop(
-                from_selector, to_selector
-            ),
-            "browser_hover": lambda selector: self.page.hover(selector),
-            "browser_select_option": lambda selector: self.page.select_option(selector),
-            "browser_mouse_click_xy": lambda x, y: self.page.mouse.click(x, y),
+    _actions: dict[str, Callable]
+    _browser: Any
+    _page: Page
+
+    def model_post_init(self, __context: Any):
+        self._actions = {
+            "browser_press_key": self.browser_press_key,
+            "browser_type": self.browser_type,
+            "browser_click": self.browser_click,
+            "browser_drag": self.browser_drag,
+            "browser_hover": self.browser_hover,
+            "browser_select_option": self.browser_select_option,
+            "browser_mouse_click_xy": self.browser_mouse_click_xy,
         }
 
-    def drag_and_drop(self, from_selector: str, to_selector: str):
-        from_elem = self.page.locator(from_selector)
+    def browser_press_key(self, key: str):
+        """
+        Press a key on the keyboard.
+        """
+        self._page.keyboard.press(key)
+
+    def browser_type(self, text: str):
+        """
+        Type text into the focused element.
+        """
+        self._page.type(text)
+
+    def browser_click(self, selector: str):
+        """
+        Click on a selector.
+        """
+        self._page.click(selector)
+
+    def browser_drag(self, from_selector: str, to_selector: str):
+        """
+        Drag and drop from one selector to another.
+        """
+        from_elem = self._page.locator(from_selector)
         from_elem.hover(timeout=500)
-        self.page.mouse.down()
+        self._page.mouse.down()
 
-        to_elem = self.page.locator(to_selector)
+        to_elem = self._page.locator(to_selector)
         to_elem.hover(timeout=500)
-        self.page.mouse.up()
+        self._page.mouse.up()
+
+    def browser_hover(self, selector: str):
+        """
+        Hover over a given element.
+        """
+        self._page.hover(selector)
+
+    def browser_select_option(self, selector: str):
+        """
+        Select an option from a given element.
+        """
+        self._page.select_option(selector)
+
+    def browser_mouse_click_xy(self, x: int, y: int):
+        """
+        Click at a given x, y coordinate using the mouse.
+        """
+        self._page.mouse.click(x, y)
 
     def initialize(self):
-        self.browser = sync_playwright().start().chromium.launch(headless=True)
-        self.page = self.browser.new_page()
+        self._browser = sync_playwright().start().chromium.launch(headless=True, chromium_sandbox=True)
+        self._page = self._browser.new_page()
 
     def run_js(self, js: str):
-        return self.page.evaluate(js)
+        js_result = self._page.evaluate(js)
+        logger.info(f"JS result: {js_result}")
+        return js_result
 
     def goto(self, url: str):
-        self.page.goto(url)
+        self._page.goto(url)
 
     def page_snapshot(self):
-        return self.page.content()
+        return self._page.content()
 
     def page_screenshot(self):
-        return self.page.screenshot()
+        scr_bytes = self._page.screenshot()
+        return Image.open(BytesIO(scr_bytes))
 
     def step(self, action: ToolCallAction):
-        fn = self.actions[action.function.name]
-        return fn(**action.function.arguments)
-
-    def actions(self):
-        return self.page.actions()
+        fn = self._actions[action.function.name]
+        action_result = fn(**action.function.arguments)
+        snapshot = self.page_snapshot()
+        screenshot = self.page_screenshot()
+        return {
+            "pruned_html": f"{action_result or ''}\n{snapshot}",
+            "axtree_txt": snapshot,
+            "screenshot": screenshot,
+        }
+    def actions(self) -> tuple[ToolSpec]:
+        specs = [ToolSpec.from_function(fn) for fn in self._actions.values()]
+        return tuple(specs)
 
     def close(self):
-        self.browser.close()
+        self._browser.close()

src/agentlab/benchmarks/miniwob/task.py

@@ -120,10 +120,14 @@ def get_task_validate_js(self) -> str:
 return [WOB_REWARD_GLOBAL, WOB_RAW_REWARD_GLOBAL, WOB_REWARD_REASON, WOB_DONE_GLOBAL, WOB_EPISODE_ID, WOB_TASK_READY];
 }"""
 
-    def parse_validation_result(self, validation_result: str) -> tuple[float, dict]:
-        chunks = [c.strip() for c in validation_result.split(",")]
+    def parse_validation_result(self, validation_result: str | list) -> tuple[float, dict]:
+        if isinstance(validation_result, list):
+            chunks = validation_result
+            done = chunks[3]
+        else:
+            chunks = [c.strip() for c in validation_result.split(",")]
+            done = chunks[3].strip().lower() == "true"
         raw_reward = float(chunks[1])
-        done = chunks[3].strip().lower() == "true"
         reward = float(raw_reward > 0)
         return reward, {
             "raw_reward": raw_reward,