Merge pull request #214 from ServiceNow/multitool_envs

Gaia bench with tape agent and multitool env

Author: recursix

.github/workflows/darglint.yml

@@ -31,4 +31,4 @@ jobs:
         run: pip list
 
       - name: Darglint checks
-        run: darglint -v 2 -z short .
+        run: darglint -v 2 -z short src/

.gitignore

@@ -3,7 +3,7 @@ __pycache__/
 *.py[cod]
 *$py.class
 results/
-.vscode
+
 # C extensions
 *.so
 # Distribution / packaging
@@ -160,11 +160,14 @@ cython_debug/
 # MacOS
 **/.DS_Store
 
-.vscode
 
 _sandbox.py
 
 results/
 
 # gradio
-.gradio/
+.gradio/
+
+outputs/
+miniwob-plusplus/
+.miniwob-server.pid

.vscode/launch.json

@@ -0,0 +1,19 @@
+{
+    // Use IntelliSense to learn about possible attributes.
+    // Hover to view descriptions of existing attributes.
+    // For more information, visit: https://go.microsoft.com/fwlink/?linkid=830387
+    "version": "0.2.0",
+    "configurations": [
+        {
+            "name": "Python Debugger: Current File",
+            "type": "debugpy",
+            "request": "launch",
+            "program": "${file}",
+            "console": "integratedTerminal",
+            "justMyCode": false,
+            "env": {
+                "AGENTLAB_DEBUG": "1"
+            }
+        }
+    ]
+}

.vscode/settings.json

@@ -0,0 +1,15 @@
+{
+    "[python]": {
+        "editor.formatOnSave": true,
+        "editor.defaultFormatter": "ms-python.black-formatter",
+        "editor.codeActionsOnSave": {
+            "source.organizeImports": "explicit",
+            "source.fixAll": "never"
+        }
+    },
+    "python.testing.pytestArgs": [
+        "tests"
+    ],
+    "python.testing.unittestEnabled": false,
+    "python.testing.pytestEnabled": true,
+}

Makefile

@@ -0,0 +1,32 @@
+.PHONY: test setup miniwob lint stop-miniwob
+
+setup:
+	@pip install -e .
+	@playwright install chromium --with-deps
+	@python -c 'import nltk; nltk.download("punkt_tab")'
+
+miniwob: stop-miniwob
+	@git clone https://github.com/Farama-Foundation/miniwob-plusplus.git || true
+	@cd miniwob-plusplus && git checkout 7fd85d71a4b60325c6585396ec4f48377d049838
+	@python -m http.server 8080 --directory miniwob-plusplus/miniwob/html & echo $$! > .miniwob-server.pid
+	@sleep 3
+	@echo "MiniWob server started on http://localhost:8080"
+
+check-miniwob:
+	@curl -I "http://localhost:8080/miniwob/" || (echo "MiniWob not reachable" && exit 1)
+	@echo "MiniWob server is reachable"
+
+stop-miniwob:
+	@kill -9 `cat .miniwob-server.pid` || true
+	@rm -f .miniwob-server.pid
+	@echo "MiniWob server stopped"
+
+run-tests:
+	@MINIWOB_URL="http://localhost:8080/miniwob/" pytest -n 5 --durations=10 -m 'not pricy' tests/
+	@echo "Tests completed"
+
+test: setup miniwob check-miniwob run-tests stop-miniwob
+
+lint: setup
+	@black src/ --check --diff
+	@darglint -v 2 -z short src/

requirements.txt

@@ -5,13 +5,15 @@ pytest==7.3.2
 flaky
 pytest-xdist
 pytest-playwright
+pydantic~=2.9
 dask
 distributed
 browsergym>=0.7.1
 joblib>=1.2.0
 openai>=1.7,<2
 langchain_community
 tiktoken
+tapeagents[converters]
 huggingface_hub
 contexttimer
 ipython
@@ -24,3 +26,4 @@ matplotlib
 ray[default]
 python-slugify
 pillow
+gymnasium>=0.27

src/agentlab/agents/README.md

@@ -99,7 +99,7 @@ have to specify the type of each field (You can use Any if it is unknown)*
 ```python
 from dataclasses import dataclass
 from browsergym.experiment.agent import Agent
-from browsergym.experiment.loop import AgentArgs
+from agentlab.experiments.loop import AgentArgs
 
 
 @dataclass
@@ -116,7 +116,7 @@ class CustomAgentArgs(AgentArgs):
 To run experiments with your custom agent, define an instance of `ExpArgs` with the required parameters.
 
 ```python
-from browsergym.experiment.loop import ExpArgs
+from agentlab.experiments.loop import ExpArgs
 
 exp_args = ExpArgs(
     agent_args=CustomAgentArgs(custom_param="value"),

src/agentlab/agents/generic_agent/reproducibility_agent.py

@@ -20,13 +20,10 @@
 
 import bgym
 from browsergym.experiments.agent import AgentInfo
-from browsergym.experiments.loop import ExpArgs, ExpResult, yield_all_exp_results
 from bs4 import BeautifulSoup
-from langchain.schema import AIMessage, BaseMessage
-from langchain_community.adapters.openai import convert_message_to_dict
 
 from agentlab.agents.agent_args import AgentArgs
-from agentlab.agents.dynamic_prompting import ActionFlags
+from agentlab.experiments.loop import ExpArgs, ExpResult, yield_all_exp_results
 from agentlab.experiments.study import Study
 from agentlab.llm.chat_api import make_assistant_message
 from agentlab.llm.llm_utils import Discussion, messages_to_dict
@@ -65,7 +62,6 @@ def get_stats(self):
 
 @dataclass
 class ReproAgentArgs(GenericAgentArgs):
-
     # starting with "_" will prevent from being part of the index in the load_results function
     _repro_dir: str = None
 
@@ -81,7 +77,6 @@ def make_agent(self):
 
 
 class ReproAgent(GenericAgent):
-
     def __init__(
         self,
         chat_model_args,
@@ -93,7 +88,6 @@ def __init__(
         super().__init__(chat_model_args, flags, max_retry)
 
     def get_action(self, obs):
-
         # replace the chat model with a reproducible chat that will mimic the
         # same answers
         step = len(self.actions)

src/agentlab/agents/most_basic_agent/most_basic_agent.py

@@ -5,7 +5,7 @@
 import bgym
 
 from agentlab.agents.agent_args import AgentArgs
-from agentlab.llm.chat_api import make_system_message, make_user_message
+from agentlab.experiments.loop import ExpArgs
 from agentlab.llm.llm_configs import CHAT_MODEL_ARGS_DICT
 from agentlab.llm.llm_utils import (
     Discussion,
@@ -133,7 +133,7 @@ def parser(response: str) -> tuple[dict, bool, str]:
 
 # example for 2 experiments testing chain of thoughts on a miniwob task
 exp_args = [
-    bgym.ExpArgs(
+    ExpArgs(
         agent_args=MostBasicAgentArgs(
             temperature=0.1,
             use_chain_of_thought=True,
@@ -142,7 +142,7 @@ def parser(response: str) -> tuple[dict, bool, str]:
         env_args=env_args,
         logging_level=logging.INFO,
     ),
-    bgym.ExpArgs(
+    ExpArgs(
         agent_args=MostBasicAgentArgs(
             temperature=0.1,
             use_chain_of_thought=False,