Skip to content
Test Suite

remove need for project id in Browserbase params if already passed to stagehand #328

Sign in to view logs

remove need for project id in Browserbase params if already passed to stagehand

remove need for project id in Browserbase params if already passed to stagehand #328

Workflow file for this run

.github/workflows/test.yml at b6cdd49
name: Test Suite
on:
push:
branches: [ main, develop ]
pull_request:
branches: [ main, develop ]
types: [opened, synchronize, reopened, labeled, unlabeled]
jobs:
test-unit:
name: Unit Tests
runs-on: ubuntu-latest
strategy:
matrix:
python-version: ["3.9", "3.10", "3.11", "3.12"]
steps:
- uses: actions/checkout@v4
- name: Set up Python ${{ matrix.python-version }}
uses: actions/setup-python@v4
with:
python-version: ${{ matrix.python-version }}
- name: Cache pip dependencies
uses: actions/cache@v3
with:
path: ~/.cache/pip
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}
restore-keys: |
${{ runner.os }}-pip-
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
# Install jsonschema for schema validation tests
pip install jsonschema
- name: Run unit tests
run: |
pytest tests/unit/ -v --junit-xml=junit-unit-${{ matrix.python-version }}.xml
- name: Upload unit test results
uses: actions/upload-artifact@v4
if: always()
with:
name: unit-test-results-${{ matrix.python-version }}
path: junit-unit-${{ matrix.python-version }}.xml
test-integration-local:
name: Local Integration Tests
runs-on: ubuntu-latest
needs: test-unit
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y xvfb
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
pip install jsonschema
# Install Playwright browsers for integration tests
playwright install chromium
playwright install-deps chromium
- name: Run local integration tests
run: |
# Run integration tests marked as 'integration' and 'local'
xvfb-run -a pytest tests/integration/ -v \
--cov=stagehand \
--cov-report=xml \
--junit-xml=junit-integration-local.xml \
-m "integration and local" \
--tb=short \
--maxfail=5
env:
MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY || 'mock-openai-key' }}
DISPLAY: ":99"
- name: Upload integration test results
uses: actions/upload-artifact@v4
if: always()
with:
name: integration-test-results-local
path: junit-integration-local.xml
- name: Upload coverage data
uses: actions/upload-artifact@v4
if: always()
with:
name: coverage-data-integration-local
path: |
.coverage
coverage.xml
test-integration-api:
name: API Integration Tests
runs-on: ubuntu-latest
needs: test-unit
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
pip install jsonschema
- name: Run API integration tests
run: |
pytest tests/integration/ -v \
--cov=stagehand \
--cov-report=xml \
--junit-xml=junit-integration-api.xml \
-m "integration and api" \
--tb=short
env:
BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }}
MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL }}
- name: Upload API integration test results
uses: actions/upload-artifact@v4
if: always()
with:
name: api-integration-test-results
path: junit-integration-api.xml
smoke-tests:
name: Smoke Tests
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
pip install jsonschema
- name: Run smoke tests
run: |
pytest tests/ -v \
--junit-xml=junit-smoke.xml \
-m "smoke" \
--tb=line \
--maxfail=5
- name: Upload smoke test results
uses: actions/upload-artifact@v4
if: always()
with:
name: smoke-test-results
path: junit-smoke.xml
test-regression:
name: Regression Tests
runs-on: ubuntu-latest
needs: test-unit
if: |
contains(github.event.pull_request.labels.*.name, 'test-regression') ||
contains(github.event.pull_request.labels.*.name, 'regression')
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y xvfb
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
pip install jsonschema
playwright install chromium
playwright install-deps chromium
- name: Run regression tests
run: |
xvfb-run -a pytest tests/ -v \
--cov=stagehand \
--cov-report=xml \
--junit-xml=junit-regression.xml \
-m "regression" \
--tb=short \
--maxfail=10
env:
BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY || 'mock-api-key' }}
BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID || 'mock-project-id' }}
MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL || 'http://localhost:3000' }}
- name: Upload regression test results
uses: actions/upload-artifact@v4
if: always()
with:
name: regression-test-results
path: junit-regression.xml
- name: Upload coverage data
uses: actions/upload-artifact@v4
if: always()
with:
name: coverage-data-regression
path: |
.coverage
coverage.xml
test-e2e:
name: End-to-End Tests
runs-on: ubuntu-latest
needs: test-unit
if: |
contains(github.event.pull_request.labels.*.name, 'test-e2e') ||
contains(github.event.pull_request.labels.*.name, 'e2e')
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Install system dependencies
run: |
sudo apt-get update
sudo apt-get install -y xvfb
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
pip install jsonschema
playwright install chromium
playwright install-deps chromium
- name: Run E2E tests
run: |
xvfb-run -a pytest tests/ -v \
--cov=stagehand \
--cov-report=xml \
--junit-xml=junit-e2e.xml \
-m "e2e" \
--tb=short
env:
BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY || 'mock-api-key' }}
BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID || 'mock-project-id' }}
MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL || 'http://localhost:3000' }}
DISPLAY: ":99"
- name: Upload E2E test results
uses: actions/upload-artifact@v4
if: always()
with:
name: e2e-test-results
path: junit-e2e.xml
test-all:
name: Complete Test Suite
runs-on: ubuntu-latest
needs: test-unit
if: |
contains(github.event.pull_request.labels.*.name, 'test-all') ||
contains(github.event.pull_request.labels.*.name, 'full-test')
steps:
- uses: actions/checkout@v4
- name: Set up Python 3.11
uses: actions/setup-python@v4
with:
python-version: "3.11"
- name: Install dependencies
run: |
python -m pip install --upgrade pip
pip install -e ".[dev]"
pip install jsonschema
playwright install chromium
- name: Run complete test suite
run: |
pytest tests/ -v \
--cov=stagehand \
--cov-report=xml \
--cov-report=html \
--junit-xml=junit-all.xml \
--maxfail=10 \
--tb=short
env:
BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }}
MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
ANTHROPIC_API_KEY: ${{ secrets.ANTHROPIC_API_KEY }}
STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL }}
- name: Upload complete test results
uses: actions/upload-artifact@v4
if: always()
with:
name: complete-test-results
path: |
junit-all.xml
htmlcov/