Add support for weblinx (#60)

* Update _get_benchmark_version to include weblinx

* Update agentxray port to use env var

* Update get_benchmark_env_args to include weblinx support

Author: xhluca

src/agentlab/analyze/agent_xray.py

@@ -1,4 +1,5 @@
 import base64
+import os
 import traceback
 from copy import deepcopy
 from io import BytesIO
@@ -481,7 +482,7 @@ def run_gradio(results_dir: Path):
         tabs.select(tab_select)
 
     demo.queue()
-    demo.launch(server_port=7899, share=True)
+    demo.launch(server_port=int(os.getenv("AGENTXRAY_APP_PORT", 7899)), share=True)
 
 
 def tab_select(evt: gr.SelectData):

src/agentlab/experiments/reproducibility_util.py

@@ -30,6 +30,11 @@ def _get_benchmark_version(benchmark_name):
         return metadata.distribution("browsergym.webarena").version
     elif benchmark_name.startswith("visualwebarena"):
         return metadata.distribution("browsergym.visualwebarena").version
+    elif benchmark_name.startswith("weblinx"):
+        try:
+            return metadata.distribution("weblinx_browsergym").version
+        except metadata.PackageNotFoundError:
+            return "0.0.1rc1"
     else:
         raise ValueError(f"Unknown benchmark {benchmark_name}")
 

src/agentlab/experiments/task_collections.py

@@ -124,6 +124,7 @@ def get_benchmark_env_args(
         "webarena": 15,
         "miniwob": 10,
         "miniwob_tiny_test": 5,
+        "weblinx": None,
     }
 
     n_repeat_default = {
@@ -133,12 +134,13 @@ def get_benchmark_env_args(
         "webarena": 1,
         "miniwob": 5,
         "miniwob_tiny_test": 2,
+        "weblinx": 1,
     }
 
     if max_steps is None:
-        max_steps = max_steps_default[benchmark_id]
+        max_steps = max_steps_default.get(benchmark_id, None)
     if n_repeat is None:
-        n_repeat = n_repeat_default[benchmark_id]
+        n_repeat = n_repeat_default.get(benchmark_id, 1)
     else:
         if benchmark_id == "webarena" and n_repeat != 1:
             logger.warning(
@@ -184,6 +186,10 @@ def get_benchmark_env_args(
         env_args_list = _make_env_args(
             miniwob_benchmarks_map[benchmark_name], max_steps, n_repeat, rng
         )
+    elif benchmark_name.startswith("weblinx"):
+        from weblinx_browsergym import ALL_WEBLINX_TASK_IDS
+
+        env_args_list = _make_env_args(ALL_WEBLINX_TASK_IDS, max_steps, n_repeat, rng)
     else:
         raise ValueError(f"Unknown benchmark name: {benchmark_name}")