diff --git a/.github/workflows/instance_pool_ci.yml b/.github/workflows/instance_pool_ci.yml new file mode 100644 index 0000000..b820ca0 --- /dev/null +++ b/.github/workflows/instance_pool_ci.yml @@ -0,0 +1,37 @@ +name: Monitor the pool of WorkArena instances + +on: + workflow_dispatch: # allow manual trigger + schedule: + - cron: "0 3 * * *" # daily at 03:00 UTC (adjust as needed) + +jobs: + browsergym-workarena-fast: + runs-on: ubuntu-22.04 + + defaults: + run: + shell: bash -l {0} + + steps: + - name: Checkout Repository + uses: actions/checkout@v4 + + - name: Set up Python + uses: actions/setup-python@v5 + with: + python-version: '3.12' + cache: 'pip' + + - name: Pip install + working-directory: ./dev + run: pip install -r requirements.txt + + - name: Pip list + run: pip list + + - name: Install Playwright + run: playwright install chromium --with-deps + + - name: Run instance checks + run: pytest -n 5 --durations=10 --slowmo 1000 -v tests/test_snow_instance.py tests/test_task_general.py diff --git a/.github/workflows/unit_tests.yml b/.github/workflows/unit_tests.yml index f5f1902..2e7d866 100644 --- a/.github/workflows/unit_tests.yml +++ b/.github/workflows/unit_tests.yml @@ -36,7 +36,7 @@ jobs: run: black . --check browsergym-workarena-fast: - runs-on: ubuntu-latest + runs-on: ubuntu-22.04 defaults: run: @@ -50,7 +50,7 @@ jobs: - name: Set up Python uses: actions/setup-python@v5 with: - python-version: '3.10' + python-version: '3.12' cache: 'pip' # caching pip dependencies - name: Pip install @@ -59,9 +59,9 @@ jobs: - name: Pip list run: pip list - + - name: Install Playwright - run: playwright install --with-deps + run: playwright install chromium --with-deps - name: Run non-slow browsergym-workarena Unit Tests env: diff --git a/tests/test_task_general.py b/tests/test_task_general.py index b410003..ab61020 100644 --- a/tests/test_task_general.py +++ b/tests/test_task_general.py @@ -15,6 +15,21 @@ from tenacity import retry, stop_after_attempt, retry_if_exception_type from browsergym.workarena import ATOMIC_TASKS +from browsergym.workarena.instance import SNowInstance, fetch_instances + +INSTANCE_POOL = fetch_instances() + +if not INSTANCE_POOL: + pytest.skip( + "No ServiceNow instances available from fetch_instances().", allow_module_level=True + ) + + +@pytest.fixture( + scope="session", params=INSTANCE_POOL, ids=[entry["url"] for entry in INSTANCE_POOL] +) +def snow_instance_entry(request): + return request.param @retry( @@ -26,8 +41,12 @@ @pytest.mark.parametrize("task_entrypoint", ATOMIC_TASKS) @pytest.mark.parametrize("random_seed", range(1)) @pytest.mark.slow -def test_cheat(task_entrypoint, random_seed: int, page: Page): - task = task_entrypoint(seed=random_seed) +def test_cheat(task_entrypoint, random_seed: int, page: Page, snow_instance_entry): + instance = SNowInstance( + snow_url=snow_instance_entry["url"], + snow_credentials=("admin", snow_instance_entry["password"]), + ) + task = task_entrypoint(seed=random_seed, instance=instance) goal, info = task.setup(page=page) chat_messages = [] reward, done, message, info = task.validate(page, chat_messages)