black formater

Author: NicolasAG

browsergym/experiments/src/browsergym/experiments/benchmark/base.py

@@ -53,7 +53,13 @@ def make_action_set(self):
 
 
 BenchmarkBackend = Literal[
-    "miniwob", "webarena", "webarena_verified", "visualwebarena", "workarena", "assistantbench", "weblinx"
+    "miniwob",
+    "webarena",
+    "webarena_verified",
+    "visualwebarena",
+    "workarena",
+    "assistantbench",
+    "weblinx",
 ]
 
 

browsergym/experiments/src/browsergym/experiments/benchmark/utils.py

@@ -158,8 +158,7 @@ def prepare_backend(backend: str):
             massage_tasks(
                 [
                     f"webarena_verified.{intent_template_id}.{task_id}"
-                    for intent_template_id, task_id in
-                    [
+                    for intent_template_id, task_id in [
                         # gitlab, shopping_admin, and map are not ready yet
                         (23, 410),  # reddit
                         # (330, 533),  # gitlab

browsergym/webarena/src/browsergym/webarena/task.py

@@ -111,7 +111,9 @@ def setup(self, page: playwright.sync_api.Page) -> tuple[str, dict]:
                     extra_headers = json.load(f)
                 page.context.set_extra_http_headers(extra_headers)
             except Exception as e:
-                logger.warning(f"Failed to load extra headers from {extra_headers_file_path}: {e}. Make sure to set the PW_EXTRA_HEADERS environment variable to the path of an existing json file containing the extra headers. Continuing without extra headers.")
+                logger.warning(
+                    f"Failed to load extra headers from {extra_headers_file_path}: {e}. Make sure to set the PW_EXTRA_HEADERS environment variable to the path of an existing json file containing the extra headers. Continuing without extra headers."
+                )
 
         # authenticate
         for site in self.config["sites"]:

browsergym/webarena_verified/src/browsergym/webarena_verified/config.py

@@ -10,4 +10,6 @@
 for task in data:
     INTENT_TEMPLATE_IDS.append(task["intent_template_id"])
 
-assert len(INTENT_TEMPLATE_IDS) == len(TASK_IDS), "Number of intent template IDs must match number of task IDs"
+assert len(INTENT_TEMPLATE_IDS) == len(
+    TASK_IDS
+), "Number of intent template IDs must match number of task IDs"

browsergym/webarena_verified/src/browsergym/webarena_verified/evaluators.py

@@ -33,7 +33,7 @@
 class WebArenaVerifiedEvaluator:
     """
     Evaluator that integrates the webarena_verified evaluation system.
-    
+
     This evaluator handles the new evaluation format with:
     - expected_retrieve_value: Validates data retrieval
     - expected_backend_state: Validates backend/database changes
@@ -57,8 +57,8 @@ def __init__(self, webarena_instance: WebArenaInstance):
                 },
                 WebArenaSite.HOMEPAGE: EnvironmentConfig(
                     urls=[webarena_instance.home_url],
-                )
-            }
+                ),
+            },
         )
         # Instantiate data reader and evaluator
         reader = WebArenaVerifiedDataReader(config)
@@ -84,6 +84,7 @@ def __call__(
         """
         # import webarena dynamically
         from webarena.browser_env.actions import ActionTypes
+
         # if last action is not a STOP action, return 0.0 as the task is not completed yet
         if trajectory[-1].get("action_type") != ActionTypes.STOP:
             return 0.0
@@ -110,9 +111,14 @@ def __call__(
         logger.info(f"Running webarena_verified evaluation for task {task.task_id}")
         results: TaskEvalResult = self.evaluator.evaluate_task(context=context)
         logger.info(f"Webarena_verified evaluation result for task {task.task_id}:")
-        logger.info(f"status: {results.status}, score: {results.score}, error_msg: {results.error_msg}")
+        logger.info(
+            f"status: {results.status}, score: {results.score}, error_msg: {results.error_msg}"
+        )
         for result in results.evaluators_results:
-            logger.info(f"- {result.evaluator_name}: status: {result.status}, score: {result.score}, error_msg: {result.error_msg}")
+            logger.info(
+                f"- {result.evaluator_name}: status: {result.status}, score: {result.score}, error_msg: {result.error_msg}"
+            )
         # return average score
-        return sum(result.score for result in results.evaluators_results) / len(results.evaluators_results)
-
+        return sum(result.score for result in results.evaluators_results) / len(
+            results.evaluators_results
+        )

browsergym/webarena_verified/src/browsergym/webarena_verified/task.py

@@ -21,10 +21,10 @@ class WebArenaVerifiedTask(GenericWebArenaTask):
     """
     WebArena Verified task class that integrates the full evaluation system
     from platform-labs-agent-eval-harness.
-    
+
     This task class handles the new evaluation format with:
     - expected_retrieve_value
-    - expected_backend_state  
+    - expected_backend_state
     - expected_ui_state
     """
 
@@ -50,7 +50,7 @@ def __init__(
             .joinpath("webarena_verified.json")
             .read_text()
         )
-        
+
         # substitute URLs
         for pattern, url_key in {
             "__GITLAB__": "gitlab",
@@ -90,9 +90,12 @@ def setup(self, page: playwright.sync_api.Page) -> tuple[str, dict]:
         # pick a task at random
         self.config = self.random.choice(self.task_configs)
 
-        # hack: dynamically build a config file to read from
+        # dynamically build a config file to read from
         with tempfile.NamedTemporaryFile(
-            mode="w+", delete=False, prefix=f"wav-{self.config['intent_template_id']}-{self.config['task_id']}_", suffix=".json"
+            mode="w+",
+            delete=False,
+            prefix=f"wav-{self.config['intent_template_id']}-{self.config['task_id']}_",
+            suffix=".json",
         ) as f:
             json.dump(self.config, f, indent=4)
             f.flush()
@@ -109,7 +112,9 @@ def setup(self, page: playwright.sync_api.Page) -> tuple[str, dict]:
                     extra_headers = json.load(f)
                 page.context.set_extra_http_headers(extra_headers)
             except Exception as e:
-                logger.warning(f"Failed to load extra headers from {extra_headers_file_path}: {e}. Make sure to set the PW_EXTRA_HEADERS environment variable to the path of an existing json file containing the extra headers. Continuing without extra headers.")
+                logger.warning(
+                    f"Failed to load extra headers from {extra_headers_file_path}: {e}. Make sure to set the PW_EXTRA_HEADERS environment variable to the path of an existing json file containing the extra headers. Continuing without extra headers."
+                )
 
         # authenticate
         for site in self.config["sites"]:

browsergym/webarenalite/src/browsergym/webarenalite/task.py

@@ -96,7 +96,9 @@ def setup(self, page: playwright.sync_api.Page) -> tuple[str, dict]:
                     extra_headers = json.load(f)
                 page.context.set_extra_http_headers(extra_headers)
             except Exception as e:
-                logger.warning(f"Failed to load extra headers from {extra_headers_file_path}: {e}. Make sure to set the PW_EXTRA_HEADERS environment variable to the path of an existing json file containing the extra headers. Continuing without extra headers.")
+                logger.warning(
+                    f"Failed to load extra headers from {extra_headers_file_path}: {e}. Make sure to set the PW_EXTRA_HEADERS environment variable to the path of an existing json file containing the extra headers. Continuing without extra headers."
+                )
 
         # authenticate
         for site in self.config["sites"]: