Merge branch 'wa_verified' of github.com:ServiceNow/BrowserGym into wa_verified

Author: NicolasAG

.github/workflows/pypi.yml

@@ -30,6 +30,9 @@ jobs:
         run: python3 -m build browsergym/webarena/ --outdir dist/
       - name: Build a binary wheel and a source tarball (browsergym-webarenalite)
         run: python3 -m build browsergym/webarenalite/ --outdir dist/
+    
+      - name: Build a binary wheel and a source tarball (browsergym-webarena-verified)
+        run: python3 -m build browsergym/webarena_verified/ --outdir dist
       - name: Build a binary wheel and a source tarball (browsergym-webarena)
         run: python3 -m build browsergym/visualwebarena/ --outdir dist/
 

browsergym/experiments/src/browsergym/experiments/benchmark/configs.py

@@ -140,13 +140,16 @@
         supports_parallel_seeds=False,
         backends=["webarena_verified"],
         env_args_list=make_env_args_list_from_repeat_tasks(
-            task_list=task_list_from_metadata(metadata=task_metadata("webarena_verified")),
+            task_list=task_list_from_metadata(
+                metadata=task_metadata("webarena_verified")
+            ),
             max_steps=30,
             n_repeats=n_repeats,
             seeds_rng=np.random.RandomState(42),
         ),
         task_metadata=task_metadata("webarena_verified"),
-    ),
+    ),  # TODO: Add webarena-verified hard subsets by filtering tasks in
+    # https://github.com/ServiceNow/webarena-verified/blob/main/assets/dataset/subsets/webarena-verified-hard.json
     "webarena_lite": lambda n_repeats=1: Benchmark(
         name="webarena_lite",
         high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
@@ -267,7 +270,8 @@
         backends=["assistantbench"],
         env_args_list=make_env_args_list_from_repeat_tasks(
             task_list=task_list_from_metadata(
-                metadata=task_metadata("assistantbench"), filter={"browsergym_split": "valid|test"}
+                metadata=task_metadata("assistantbench"),
+                filter={"browsergym_split": "valid|test"},
             ),
             max_steps=30,
             n_repeats=n_repeats,

browsergym/webarena_verified/src/browsergym/webarena_verified/config.py

@@ -12,7 +12,7 @@
 # Check if the json file is the same as the one in the webarena-verified repository
 library_json_string = (
     importlib.resources.files("webarena_verified")
-    .joinpath("../../assets/dataset/webarena-verified.json")
+    .joinpath("assets/dataset/webarena-verified.json")
     .read_text()
 )
 library_json = json.loads(library_json_string)