fixes, use firefox

Author: ollmer

src/agentlab/actions.py

@@ -0,0 +1,47 @@
+from bgym import AbstractActionSet
+from tapeagents.tool_calling import FunctionCall, ToolCallAction, ToolSpec
+
+from agentlab.llm.llm_utils import parse_html_tags_raise
+
+
+class ToolsActionSet(AbstractActionSet):
+    def __init__(self, actions:list[ToolSpec]):
+        self.actions = actions
+
+    def describe(self, with_long_description: bool = True, with_examples: bool = True) -> str:
+        tools_description = "\n".join([action.description() for action in self.actions])
+        return tools_description
+
+    def example_action(self, abstract: bool) -> str:
+        if abstract:
+            return """<action>
+{
+    "name": "<action_name>",
+    "arguments": {
+        "<argument_name_1>": "<argument_value_1>",
+        "<argument_name_2>": "<argument_value_2>",
+        ...
+    }
+}
+</action>
+"""
+        else:
+            return """<action>
+{
+    "name": "browser_navigate",
+    "arguments": {
+        "url": "https://www.google.com"
+    }
+}
+</action>
+"""
+    @classmethod
+    def parse_action(cls, llm_output: str) -> ToolCallAction:
+        content_dict, valid, retry_message = parse_html_tags_raise(llm_output, keys=["action"])
+        if not valid or "action" not in content_dict:
+            raise ValueError(f"Invalid action: llm_output: {llm_output}, retry_message: {retry_message}")
+        action_str = content_dict["action"]
+        return ToolCallAction(function=FunctionCall(name=action_str["name"], arguments=action_str["arguments"]))
+
+    def to_python_code(self, action) -> str:
+        return action.model_dump_json(indent=2)

src/agentlab/agents/tapeagent/agent.py

@@ -41,12 +41,12 @@ def load_config(config_name: str) -> DictConfig:
 class TapeAgentArgs(AgentArgs):
     config: DictConfig = None  # type: ignore
 
-    def make_agent(self, known_actions: tuple[ToolSpec, ...] | None) -> bgym.Agent:
-        if known_actions is None:
+    def make_agent(self, actions: tuple[ToolSpec, ...] | None) -> bgym.Agent:
+        if actions is None:
             agent = hydra.utils.instantiate(self.config.agent)
         else:
-            tools_description = "\n".join([action.description() for action in known_actions])
-            agent = hydra.utils.instantiate(self.config.agent, known_actions=known_actions, tools_description=tools_description)
+            tools_description = "\n".join([action.description() for action in actions])
+            agent = hydra.utils.instantiate(self.config.agent, known_actions=actions, tools_description=tools_description)
         return TapeAgent(agent=agent)
 
 

src/agentlab/backends/browser/env.py

@@ -7,6 +7,7 @@
 from tapeagents.core import Action, Observation, StopStep
 from tapeagents.tool_calling import ToolCallAction, ToolSpec
 
+from agentlab.actions import ToolsActionSet
 from agentlab.backends.browser.base import BrowserBackend
 from agentlab.benchmarks.abstract_env import AbstractEnv, AbstractEnvArgs
 from agentlab.benchmarks.miniwob.task import AbstractWebTask
@@ -41,7 +42,9 @@ def reset(self, seed: int):
             logger.info(f"Task reset result: {js_result_str}")
         return [GoalObservation(goal=js_result_str), PageObservation(content=page_content)], {}
 
-    def step(self, action: ToolCallAction) -> tuple[Observation, float, bool, bool, dict]:
+    def step(self, action: ToolCallAction | str) -> tuple[Observation, float, bool, bool, dict]:
+        if isinstance(action, str):
+            action = ToolsActionSet.parse_action(action)
         logger.info(f"BrowserEnv.step() called with action {action.function.name}")
 
         action_exec_start = time.time()

src/agentlab/backends/browser/mcp_playwright.json

@@ -5,7 +5,7 @@
       "args": [
         "@playwright/mcp@latest",
         "--browser",
-        "chromium",
+        "firefox",
         "--headless",
         "--isolated"
       ],

src/agentlab/backends/browser/mcp_playwright.py

@@ -14,9 +14,13 @@ class MCPPlaywright(MCPBrowserBackend):
 
     def run_js(self, js: str):
         raw_response = self.call_tool("browser_evaluate", {"function": js})
-        _, half_response = raw_response.split("### Result", maxsplit=1)
-        result_str, _ = half_response.split("\n### Ran", maxsplit=1)
-        result_str = result_str.strip()
+        try:
+            _, half_response = raw_response.split("### Result", maxsplit=1)
+            result_str, _ = half_response.split("\n### Ran", maxsplit=1)
+            result_str = result_str.strip()
+        except Exception as e:
+            logger.error(f"Error parsing JS result: {e}. Raw result: {raw_response}")
+            raise e
         return result_str
 
     def step(self, action: ToolCallAction) -> str:

src/agentlab/experiments/loop.py

@@ -419,7 +419,7 @@ def run(self):
             if isinstance(self.env_args, BrowserEnvArgs):
                 env = self.env_args.make_env(exp_dir=self.exp_dir)
                 logger.debug("Environment created.")
-                agent = self.agent_args.make_agent(known_actions=env.actions())
+                agent = self.agent_args.make_agent(actions=env.actions())
                 logger.debug(f"Agent created with actions: {env.actions()}")
             else:
                 agent = self.agent_args.make_agent()