Merge pull request #116 from ServiceNow/instance-pool-ci

Continuous integration of instances in the pool

Author: aldro61

.github/workflows/instance_pool_ci.yml

@@ -0,0 +1,37 @@
+name: Monitor the pool of WorkArena instances
+
+on:
+  workflow_dispatch:          # allow manual trigger
+  schedule:
+    - cron: "0 3 * * *"       # daily at 03:00 UTC (adjust as needed)
+
+jobs:
+  browsergym-workarena-fast:
+    runs-on: ubuntu-22.04
+
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Checkout Repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+
+      - name: Pip install
+        working-directory: ./dev
+        run: pip install -r requirements.txt
+
+      - name: Pip list
+        run: pip list
+
+      - name: Install Playwright
+        run: playwright install chromium --with-deps
+
+      - name: Run instance checks
+        run: pytest -n 5 --durations=10 --slowmo 1000 -v tests/test_snow_instance.py tests/test_task_general.py

.github/workflows/unit_tests.yml

@@ -36,7 +36,7 @@ jobs:
         run: black . --check
 
   browsergym-workarena-fast:
-    runs-on: ubuntu-latest
+    runs-on: ubuntu-22.04
 
     defaults:
       run:
@@ -50,7 +50,7 @@ jobs:
       - name: Set up Python
         uses: actions/setup-python@v5
         with:
-          python-version: '3.10'
+          python-version: '3.12'
           cache: 'pip' # caching pip dependencies
 
       - name: Pip install
@@ -59,9 +59,9 @@ jobs:
 
       - name: Pip list
         run: pip list
-  
+        
       - name: Install Playwright
-        run: playwright install --with-deps
+        run: playwright install chromium --with-deps
 
       - name: Run non-slow browsergym-workarena Unit Tests
         env:

tests/test_task_general.py

@@ -15,6 +15,21 @@
 from tenacity import retry, stop_after_attempt, retry_if_exception_type
 
 from browsergym.workarena import ATOMIC_TASKS
+from browsergym.workarena.instance import SNowInstance, fetch_instances
+
+INSTANCE_POOL = fetch_instances()
+
+if not INSTANCE_POOL:
+    pytest.skip(
+        "No ServiceNow instances available from fetch_instances().", allow_module_level=True
+    )
+
+
+@pytest.fixture(
+    scope="session", params=INSTANCE_POOL, ids=[entry["url"] for entry in INSTANCE_POOL]
+)
+def snow_instance_entry(request):
+    return request.param
 
 
 @retry(
@@ -26,8 +41,12 @@
 @pytest.mark.parametrize("task_entrypoint", ATOMIC_TASKS)
 @pytest.mark.parametrize("random_seed", range(1))
 @pytest.mark.slow
-def test_cheat(task_entrypoint, random_seed: int, page: Page):
-    task = task_entrypoint(seed=random_seed)
+def test_cheat(task_entrypoint, random_seed: int, page: Page, snow_instance_entry):
+    instance = SNowInstance(
+        snow_url=snow_instance_entry["url"],
+        snow_credentials=("admin", snow_instance_entry["password"]),
+    )
+    task = task_entrypoint(seed=random_seed, instance=instance)
     goal, info = task.setup(page=page)
     chat_messages = []
     reward, done, message, info = task.validate(page, chat_messages)