Merge pull request #111 from ServiceNow/new-dispatcher

Managed Instance Dispatcher

Author: aldro61

README.md

@@ -41,40 +41,26 @@ https://github.com/ServiceNow/WorkArena/assets/2374980/68640f09-7d6f-4eb1-b556-c
 
 ## Getting Started
 
-To setup WorkArena, you will need to get your own ServiceNow instance, install our Python package, and upload some data to your instance. Follow the steps below to achieve this.
+To setup WorkArena, you will need to gain access to ServiceNow instances and install our Python package locally. Follow the steps below to achieve this.
 
-### a) Create a ServiceNow Developer Instance
+### a) Gain Access to ServiceNow Instances
 
-1. Go to https://developer.servicenow.com/ and create an account.
-2. Click on `Request an instance` and select the `Washington` release (initializing the instance will take a few minutes)
-   * **Important note:** ServiceNow no longer provides Washington instances to the public. Please fill out [this form](forms.gle/6WLf6s9nkA5D8aue6) to request one.
-4. Once the instance is ready, you should see your instance URL and credentials. If not, click _Return to the Developer Portal_, then navigate to _Manage instance password_ and click _Reset instance password_.
-5. Change the role of the user to admin in yoyr instance parameters ![image](https://github.com/user-attachments/assets/6f0fbf8e-f40f-411a-84cb-fead93d85f60)
+1. Navigate to https://huggingface.co/datasets/ServiceNow/WorkArena-Instances.
+2. Fill the form, accept the terms to gain access to the gated repository and wait for approval.
+3. Ensure that the machine where you will run WorkArena is [authenticated with Hugging Face](https://huggingface.co/docs/hub/en/datasets-polars-auth) (e.g., via huggingface-cli login or the HUGGING_FACE_HUB_TOKEN environment variable).
 
-6. You should now see your URL and credentials. Based on this information, set the following environment variables:
-    * `SNOW_INSTANCE_URL`: The URL of your ServiceNow developer instance
-    * `SNOW_INSTANCE_UNAME`: The username, should be "admin"
-    * `SNOW_INSTANCE_PWD`: The password, make sure you place the value in quotes "" and be mindful of [escaping special shell characters](https://onlinelinuxtools.com/escape-shell-characters). Running `echo $SNOW_INSTANCE_PWD` should print the correct password.
-7. Log into your instance via a browser using the admin credentials. Close any popup that appears on the main screen (e.g., agreeing to analytics).
-
-**Warning:** Feel free to look around the platform, but please make sure you revert any changes (e.g., changes to list views, pinning some menus, etc.) as these changes will be persistent and affect the benchmarking process.
-
-### b) Install WorkArena and Initialize your Instance
+### b) Install WorkArena
 
 Run the following command to install WorkArena in the [BrowswerGym](https://github.com/servicenow/browsergym) environment:
 ```
-pip install browsergym
+pip install browsergym-workarena
 ```
 
 Then, install [Playwright](https://github.com/microsoft/playwright):
 ```
 playwright install
 ```
 
-Finally, run this command in a terminal to upload the benchmark data to your ServiceNow instance:
-```
-workarena-install
-```
 Your installation is now complete! 🎉
 
 

requirements.txt

@@ -5,3 +5,4 @@ numpy>=1.14
 requests>=2.31
 tenacity>=8.2.3  # only used in cheat() -> move to tests?
 tqdm>=4.66.2
+huggingface_hub>=0.23

src/browsergym/workarena/config.py

@@ -11,6 +11,12 @@
 SNOW_JS_UTILS_FILEPATH = str(resources.files(utils).joinpath("js_utils.js"))
 SNOW_SUPPORTED_RELEASES = ["washingtondc"]
 
+# Hugging Face dataset containing available instances
+INSTANCE_REPO_ID = "ServiceNow/WorkArena-Instances"
+INSTANCE_REPO_FILENAME = "instances.json"
+INSTANCE_REPO_TYPE = "dataset"
+INSTANCE_XOR_SEED = "x3!+-9mi#nhlo%a02$9hna{]"
+
 # Path to the Menu navigation task configuration
 ALL_MENU_PATH = str(resources.files(data_files).joinpath("task_configs/all_menu.json"))
 

src/browsergym/workarena/install.py

@@ -1,3 +1,6 @@
+from __future__ import annotations
+
+import argparse
 import html
 import json
 import logging
@@ -48,10 +51,26 @@
     UI_THEMES_UPDATE_SET,
 )
 from .api.user import set_user_preference
-from .instance import SNowInstance
+from .instance import SNowInstance as _BaseSNowInstance
 from .utils import url_login
 
 
+_CLI_INSTANCE_URL: str | None = None
+_CLI_INSTANCE_PASSWORD: str | None = None
+
+
+def SNowInstance():
+    """
+    Wrapper around the standard SNowInstance that always uses CLI-provided credentials.
+    """
+    if not _CLI_INSTANCE_URL or not _CLI_INSTANCE_PASSWORD:
+        raise RuntimeError("Installer requires --instance-url and --instance-password arguments.")
+    return _BaseSNowInstance(
+        snow_url=_CLI_INSTANCE_URL,
+        snow_credentials=("admin", _CLI_INSTANCE_PASSWORD),
+    )
+
+
 def _is_dev_portal_instance() -> bool:
     """
     Check if the instance is a ServiceNow Developer Portal instance.
@@ -1105,6 +1124,23 @@ def main():
     Entrypoint for package CLI installation command
 
     """
+    parser = argparse.ArgumentParser(
+        description="Install WorkArena artifacts on a ServiceNow instance."
+    )
+    parser.add_argument(
+        "--instance-url", required=True, help="URL of the target ServiceNow instance."
+    )
+    parser.add_argument(
+        "--instance-password",
+        required=True,
+        help="Password for the admin user on the target ServiceNow instance.",
+    )
+    args = parser.parse_args()
+
+    global _CLI_INSTANCE_URL, _CLI_INSTANCE_PASSWORD
+    _CLI_INSTANCE_URL = args.instance_url
+    _CLI_INSTANCE_PASSWORD = args.instance_password
+
     logging.basicConfig(level=logging.INFO)
 
     try:

src/browsergym/workarena/instance.py

@@ -1,12 +1,88 @@
+import base64
 import json
+import logging
 import os
+import random
 import requests
-import re
+from itertools import cycle
 
+from huggingface_hub import hf_hub_download
+from huggingface_hub.utils import disable_progress_bars
 from playwright.sync_api import sync_playwright
 from typing import Optional
 
-from .config import SNOW_BROWSER_TIMEOUT, REPORT_FILTER_PROPERTY
+from .config import (
+    INSTANCE_REPO_FILENAME,
+    INSTANCE_REPO_ID,
+    INSTANCE_REPO_TYPE,
+    INSTANCE_XOR_SEED,
+    REPORT_FILTER_PROPERTY,
+    SNOW_BROWSER_TIMEOUT,
+)
+
+
+# Required to read the instance credentials
+if not INSTANCE_XOR_SEED:
+    raise ValueError("INSTANCE_XOR_SEED must be configured")
+
+
+def _xor_cipher(data: bytes, key: bytes) -> bytes:
+    return bytes(b ^ k for b, k in zip(data, cycle(key)))
+
+
+def decrypt_instance_password(encrypted_password: str) -> str:
+    """Decrypt a base64-encoded XOR-obfuscated password using the shared key."""
+
+    cipher_bytes = base64.b64decode(encrypted_password)
+    plain_bytes = _xor_cipher(cipher_bytes, INSTANCE_XOR_SEED.encode("utf-8"))
+    return plain_bytes.decode("utf-8")
+
+
+def encrypt_instance_password(password: str) -> str:
+    """Helper to produce encrypted passwords for populating the instance file."""
+
+    cipher_bytes = _xor_cipher(password.encode("utf-8"), INSTANCE_XOR_SEED.encode("utf-8"))
+    return base64.b64encode(cipher_bytes).decode("utf-8")
+
+
+def fetch_instances():
+    """
+    Load the latest instances from either a custom pool (SNOW_INSTANCE_POOL env var) or the gated HF dataset.
+    """
+    pool_path = os.getenv("SNOW_INSTANCE_POOL")
+    if pool_path:
+        path = os.path.expanduser(pool_path)
+        if not os.path.exists(path):
+            raise FileNotFoundError(
+                f"SNOW_INSTANCE_POOL points to '{pool_path}', but the file does not exist."
+            )
+        logging.info("Loading ServiceNow instances from custom pool: %s", path)
+    else:
+        try:
+            disable_progress_bars()
+            path = hf_hub_download(
+                repo_id=INSTANCE_REPO_ID,
+                filename=INSTANCE_REPO_FILENAME,
+                repo_type=INSTANCE_REPO_TYPE,
+            )
+            logging.info("Loaded ServiceNow instances from the default instance pool.")
+        except Exception as e:
+            raise RuntimeError(
+                f"Could not access {INSTANCE_REPO_ID}/{INSTANCE_REPO_FILENAME}. "
+                "Make sure you have been granted access to the gated repo and that you are "
+                "authenticated (run `huggingface-cli login` or set HUGGING_FACE_HUB_TOKEN)."
+            ) from e
+
+    with open(path, "r", encoding="utf-8") as f:
+        entries = json.load(f)
+
+    for entry in entries:
+        entry["url"] = entry["u"]
+        entry["password"] = decrypt_instance_password(entry["p"])
+        del entry["u"]
+        del entry["p"]
+
+    return entries
 
 
 class SNowInstance:
@@ -26,31 +102,39 @@ def __init__(
         Parameters:
         -----------
         snow_url: str
-            The URL of a SNow instance. If None, will try to get the value from the environment variable SNOW_INSTANCE_URL.
+            The URL of a SNow instance. When omitted, the constructor first looks for SNOW_INSTANCE_URL and falls back
+            to a random instance from the benchmark's instance pool if the environment variable is not set.
         snow_credentials: (str, str)
-            The username and password used to access the SNow instance. If None, will try to get the values from the
-            environment variables SNOW_INSTANCE_UNAME and SNOW_INSTANCE_PWD.
+            The username and password used to access the SNow instance. When omitted, environment variables
+            SNOW_INSTANCE_UNAME/SNOW_INSTANCE_PWD are used if set; otherwise, a random instance from the benchmark's
+            instance pool is selected.
 
         """
         # try to get these values from environment variables if not provided
-        if snow_url is None:
-            if "SNOW_INSTANCE_URL" in os.environ:
+        if snow_url is None or snow_credentials is None:
+
+            # Check if all required environment variables are set and if yes, fetch url and credentials from there
+            if (
+                "SNOW_INSTANCE_URL" in os.environ
+                and "SNOW_INSTANCE_UNAME" in os.environ
+                and "SNOW_INSTANCE_PWD" in os.environ
+            ):
                 snow_url = os.environ["SNOW_INSTANCE_URL"]
-            else:
-                raise ValueError(
-                    f"Please provide a ServiceNow instance URL (you can use the environment variable SNOW_INSTANCE_URL)"
-                )
-
-        if snow_credentials is None:
-            if "SNOW_INSTANCE_UNAME" in os.environ and "SNOW_INSTANCE_PWD" in os.environ:
                 snow_credentials = (
                     os.environ["SNOW_INSTANCE_UNAME"],
                     os.environ["SNOW_INSTANCE_PWD"],
                 )
+
+            # Otherwise, load all instances and select one randomly
             else:
-                raise ValueError(
-                    f"Please provide ServiceNow credentials (you can use the environment variables SNOW_INSTANCE_UNAME and SNOW_INSTANCE_PWD)"
-                )
+                instances = fetch_instances()
+                if not instances:
+                    raise ValueError(
+                        f"No instances found in the dataset {INSTANCE_REPO_ID}. Please provide instance details via parameters or environment variables."
+                    )
+                instance = random.choice(instances)
+                snow_url = instance["url"]
+                snow_credentials = ("admin", instance["password"])
 
         # remove trailing slashes in the URL, if any
         self.snow_url = snow_url.rstrip("/")

src/browsergym/workarena/tasks/dashboard.py

@@ -29,15 +29,6 @@
 #      - We currently don't support maps because they are clickable and would require a more evolved cheat function
 SUPPORTED_PLOT_TYPES = ["area", "bar", "column", "line", "pie", "spline"]
 
-# Get report filter config
-config = SNowInstance().report_filter_config
-if config is None:
-    REPORT_DATE_FILTER = REPORT_TIME_FILTER = None
-else:
-    REPORT_DATE_FILTER = config["report_date_filter"]
-    REPORT_TIME_FILTER = config["report_time_filter"]
-del config
-
 
 class DashboardRetrievalTask(AbstractServiceNowTask, ABC):
     """
@@ -303,14 +294,28 @@ def get_init_scripts(self) -> List[str]:
             """,
         ]
 
-    def setup_goal(self, page: playwright.sync_api.Page) -> Tuple[str | dict]:
-        super().setup_goal(page=page)
+    def _get_filter_config(self) -> str:
+        # Get report filter config
+        config = self.instance.report_filter_config
+        if config is None:
+            REPORT_DATE_FILTER = REPORT_TIME_FILTER = None
+        else:
+            REPORT_DATE_FILTER = config["report_date_filter"]
+            REPORT_TIME_FILTER = config["report_time_filter"]
+        del config
 
         # Check that the report filters are properly setup
         if REPORT_DATE_FILTER is None or REPORT_TIME_FILTER is None:
             raise RuntimeError(
                 "The report date and time filters are not set. Please run the install script to set them."
             )
+        return REPORT_DATE_FILTER, REPORT_TIME_FILTER
+
+    def setup_goal(self, page: playwright.sync_api.Page) -> Tuple[str | dict]:
+        super().setup_goal(page=page)
+
+        # Get the instance report filter config
+        REPORT_DATE_FILTER, REPORT_TIME_FILTER = self._get_filter_config()
 
         # Configure task
         # ... sample a configuration
@@ -632,6 +637,9 @@ def _generate_random_config(
             The types of questions to sample from (uniformely)
 
         """
+        # Get the instance report filter config
+        REPORT_DATE_FILTER, REPORT_TIME_FILTER = self._get_filter_config()
+
         # Check that the report filters are properly setup
         if REPORT_DATE_FILTER is None or REPORT_TIME_FILTER is None:
             raise RuntimeError(

src/browsergym/workarena/tasks/scripts/generate_dashboard_configs.py

@@ -15,7 +15,6 @@
 
 from browsergym.workarena.api.utils import table_api_call, table_column_info
 from browsergym.workarena.config import (
-    REPORT_DATE_FILTER,
     REPORT_PATCH_FLAG,
     REPORT_RETRIEVAL_MINMAX_CONFIG_PATH,
     REPORT_RETRIEVAL_VALUE_CONFIG_PATH,
@@ -44,6 +43,10 @@ def all_configs(self):
 
 
 def get_report_urls(instance):
+    # Get the instance report filter config
+    REPORT_DATE_FILTER, REPORT_TIME_FILTER = instance._get_filter_config()
+    raise NotImplementedError("TODO: Include the time filter as in dashboard.py")
+
     # Generate a bunch of reports on the fly based on valid table fields
     ON_THE_FLY_REPORTS = []
     for table in [
@@ -226,7 +229,13 @@ def get_all_configs_by_url(url, is_report):
 
 
 if __name__ == "__main__":
-    instance = SNowInstance()
+
+    # XXX: Make sure to specific the exact instance on which to generate configs (and not use a random one)
+    raise NotImplementedError(
+        "Make sure to specific instance URL and credentials below, then comment this line."
+    )
+    instance = SNowInstance(snow_url=None, snow_credentials=None)
+
     reports = get_report_urls(instance)
     gen_func = partial(get_all_configs_by_url, is_report=REPORT)
 

…ym/workarena/tasks/scripts/navigation.py …sks/scripts/generate_navigation_tasks.pysrc/browsergym/workarena/tasks/scripts/navigation.py renamed to src/browsergym/workarena/tasks/scripts/generate_navigation_tasks.py src/browsergym/workarena/tasks/scripts/navigation.py renamed to src/browsergym/workarena/tasks/scripts/generate_navigation_tasks.py

@@ -6,7 +6,10 @@
 
 
 def get_all_impersonation_users():
-    instance = SNowInstance()
+    raise NotImplementedError(
+        "Make sure to specific instance URL and credentials below, then comment this line."
+    )
+    instance = SNowInstance(snow_url=None, snow_credentials=None)
     candidate_users = [
         u["first_name"] + " " + u["last_name"]
         for u in table_api_call(

src/browsergym/workarena/tasks/scripts/knowledge.py

@@ -1,3 +1,4 @@
+# TODO: Can we delete this file?
 import json
 import random
 
@@ -9,9 +10,8 @@
 from tqdm import tqdm
 
 
-def generate_all_kb_configs(instance=None, num_configs=1000) -> list[dict]:
+def generate_all_kb_configs(instance, num_configs=1000) -> list[dict]:
     """Generate all possible KB configs"""
-    instance = instance if instance is not None else SNowInstance()
     with open(KB_FILEPATH, "r") as f:
         kb_entries = json.load(f)
     all_configs = []
@@ -33,5 +33,7 @@ def generate_all_kb_configs(instance=None, num_configs=1000) -> list[dict]:
 
 
 if __name__ == "__main__":
-
-    validate_kb_configs()
+    raise NotImplementedError(
+        "Make sure to specific instance URL and credentials below, then comment this line."
+    )
+    generate_all_kb_configs(instance=SNowInstance(snow_url=None))