v0.2.1 (#8)

* fix problematic task configs

* version bump

* missing author

* black config

* README update

* unit tests fix

* pypi workflow update

* README update with dashboard tasks

Author: gasse

.github/workflows/pypi.yml

@@ -48,10 +48,11 @@ jobs:
           uses: pypa/gh-action-pypi-publish@release/v1
 
     github-release:
-      name: Sign with Sigstore and upload them to GitHub Release
+      name: Sign packages with Sigstore and upload them to GitHub Release
       needs:
       - publish-to-pypi
       runs-on: ubuntu-latest
+
       permissions:
         contents: write  # IMPORTANT: mandatory for making GitHub Releases
         id-token: write  # IMPORTANT: mandatory for sigstore
@@ -64,7 +65,7 @@ jobs:
           path: dist/
 
       - name: Sign the dists with Sigstore
-        uses: sigstore/gh-action-sigstore-python@v1.2.3
+        uses: sigstore/gh-action-sigstore-python@v2.1.1
         with:
           inputs: >-
             ./dist/*.tar.gz

.github/workflows/unit_tests.yml

@@ -34,38 +34,39 @@ jobs:
         run: black . --check
 
   browsergym-workarena-fast:
-      runs-on: ubuntu-latest
+    runs-on: ubuntu-latest
 
-      defaults:
-        run:
-          shell: bash -l {0}
+    defaults:
+      run:
+        shell: bash -l {0}
 
-      steps:
+    steps:
 
-        - name: Checkout Repository
-          uses: actions/checkout@v4
+      - name: Checkout Repository
+        uses: actions/checkout@v4
 
-        - name: Set up Python
-          uses: actions/setup-python@v5
-          with:
-            python-version: '3.10'
-            cache: 'pip' # caching pip dependencies
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+          cache: 'pip' # caching pip dependencies
 
-        - name: Pip install
-          run: pip install -r requirements.txt
+      - name: Pip install
+        working-directory: ./dev
+        run: pip install -r requirements.txt
 
-        - name: Pip list
-          run: pip list
+      - name: Pip list
+        run: pip list
 
-        - name: Install Playwright
-          run: playwright install --with-deps
+      - name: Install Playwright
+        run: playwright install --with-deps
 
-        - name: Run non-slow browsergym-workarena Unit Tests
-          env:
-            SNOW_INSTANCE_URL: ${{ secrets.SNOW_INSTANCE_URL }}
-            SNOW_INSTANCE_UNAME: ${{ secrets.SNOW_INSTANCE_UNAME }}
-            SNOW_INSTANCE_PWD: ${{ secrets.SNOW_INSTANCE_PWD }}
-          run: pytest -n 5 --durations=10 -m 'not slow and not pricy' --slowmo 1000 -v tests
+      - name: Run non-slow browsergym-workarena Unit Tests
+        env:
+          SNOW_INSTANCE_URL: ${{ secrets.SNOW_INSTANCE_URL }}
+          SNOW_INSTANCE_UNAME: ${{ secrets.SNOW_INSTANCE_UNAME }}
+          SNOW_INSTANCE_PWD: ${{ secrets.SNOW_INSTANCE_PWD }}
+        run: pytest -n 5 --durations=10 -m 'not slow and not pricy' --slowmo 1000 -v tests
 
   browsergym-workarena-slow:
     runs-on: ubuntu-latest
@@ -86,6 +87,7 @@ jobs:
           cache: 'pip' # caching pip dependencies
 
       - name: Pip install
+        working-directory: ./dev
         run: pip install -r requirements.txt
 
       - name: Pip list

README.md

@@ -11,11 +11,11 @@ WorkArena is included in [BrowserGym](https://github.com/ServiceNow/BrowserGym),
 https://github.com/ServiceNow/WorkArena/assets/2374980/68640f09-7d6f-4eb1-b556-c294a6afef70
 
 ## ⚠️ Pre-Release warning ⚠️
-Please note that the WorkArena benchmark is still undergoing minor bug fixes and updates, which may cause discrepancies with results reported in our latest arXiv preprint. We plan to release soon a stable version of WorkArena v0.1.0 with enhanced stability, and a final version v1.0.0 with a new suite of tasks.
+Please note that the WorkArena benchmark is still undergoing minor bug fixes and updates, which may cause discrepancies with results reported in our latest arXiv preprint. We plan to release soon a stable version of WorkArena with enhanced stability, and a final version v1.0.0 with a new suite of tasks.
 
 ## Benchmark Contents
 
-At the moment, WorkArena includes `18,050` task instances drawn from `29` tasks that cover the main components of the ServiceNow user interface. The following videos show an agent built on `GPT-4-vision` interacting with every such component. As emphasized by our results, this benchmark is not solved and thus, the performance of the agent is not always on point.
+At the moment, WorkArena includes `18,050` task instances drawn from `33` tasks that cover the main components of the ServiceNow user interface. The following videos show an agent built on `GPT-4-vision` interacting with every such component. As emphasized by our results, this benchmark is not solved and thus, the performance of the agent is not always on point.
 
 ### Knowledge Bases
 
@@ -51,14 +51,20 @@ https://github.com/ServiceNow/WorkArena/assets/1726818/7538b3ef-d39b-4978-b9ea-8
 
 https://github.com/ServiceNow/WorkArena/assets/1726818/ca26dfaf-2358-4418-855f-80e482435e6e
 
+### Dashboards
+
+**Goal:** The agent must extract information from a dashboard.
+
+
+
 ## Getting Started
 
 To setup WorkArena, you will need to get your own ServiceNow instance, install our Python package, and upload some data to your instance. Follow the steps below to achieve this.
 
 ### a) Create a ServiceNow Developer Instance
 
 1. Go to https://developer.servicenow.com/ and create an account.
-2. Click on `Request an instance` and select the `Utah` release (initializing the instance will take a few minutes)
+2. Click on `Request an instance` and select the `Washington` release (initializing the instance will take a few minutes)
 3. Once the instance is ready, you should see your instance URL and credentials. If not, click _Return to the Developer Portal_, then navigate to _Manage instance password_ and click _Reset instance password_.
 4. You should now see your URL and credentials. Based on this information, set the following environment variables:
     * `SNOW_INSTANCE_URL`: The URL of your ServiceNow developer instance

dev/environment.yaml

@@ -0,0 +1,13 @@
+name: workarena-dev
+
+channels:
+  - huggingface
+  - conda-forge
+  - defaults
+
+dependencies:
+  - python>=3.10
+  - pip
+
+  - pip:
+      - -r requirements.txt

dev/requirements.txt

@@ -0,0 +1,9 @@
+black[jupyter]==24.2.0
+blacken-docs
+pre-commit
+pytest==7.3.2
+pytest-xdist
+pytest-playwright
+tenacity
+browsergym-core
+-e .. # local package

pyproject.toml

@@ -11,6 +11,7 @@ authors = [
     {name = "Maxime Gasse"},
     {name = "Alex Lacoste"},
     {name = "Manuel Del Verme"},
+    {name = "Megh Thakkar"},
 ]
 readme = "README.md"
 requires-python = ">3.7"
@@ -39,3 +40,30 @@ files = ["requirements.txt"]
 
 [tool.hatch.build.targets.wheel]
 packages = ["src/browsergym"]
+
+[tool.black]
+line-length = 100
+include = '\.pyi?$'
+exclude = '''
+/(
+    \.eggs
+  | \.git
+  | \.hg
+  | \.mypy_cache
+  | \.nox
+  | \.tox
+  | \.venv
+  | _build
+  | buck-out
+  | build
+  | dist
+)/
+'''
+
+[tool.pytest.ini_options]
+filterwarnings = [
+    'ignore::UserWarning:gymnasium.*:',  # too many "The obs is not within the observation space." warnings.
+]
+markers = [
+    "slow: marks tests as slow (deselect with '-m \"not slow\"')",
+]

requirements.txt

@@ -1,4 +1,4 @@
-browsergym-core==0.2.0
+browsergym-core>=0.2
 english-words>=2.0.1
 faker>=24.11.0
 numpy>=1.14

src/browsergym/workarena/__init__.py

@@ -1,4 +1,4 @@
-__version__ = "0.2.0"
+__version__ = "0.2.1"
 
 from browsergym.core.registration import register_task