Skip to content

Commit 957dbeb

Browse files
filip-michalskyfilip-michalsky
committed
one shot test structure
1 parent 9ee8096 commit 957dbeb

20 files changed

+8448
-6
lines changed

‎.github/workflows/test.yml‎

Lines changed: 331 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,331 @@
1+
name: Test Suite
2+
3+
on:
4+
push:
5+
branches: [ main, develop ]
6+
pull_request:
7+
branches: [ main, develop ]
8+
# schedule:
9+
# # Run tests daily at 6 AM UTC
10+
# - cron: '0 6 * * *'
11+
12+
jobs:
13+
test-unit:
14+
name: Unit Tests
15+
runs-on: ubuntu-latest
16+
strategy:
17+
matrix:
18+
python-version: ["3.9", "3.10", "3.11", "3.12"]
19+
20+
steps:
21+
- uses: actions/checkout@v4
22+
23+
- name: Set up Python ${{ matrix.python-version }}
24+
uses: actions/setup-python@v4
25+
with:
26+
python-version: ${{ matrix.python-version }}
27+
28+
- name: Cache pip dependencies
29+
uses: actions/cache@v3
30+
with:
31+
path: ~/.cache/pip
32+
key: ${{ runner.os }}-pip-${{ hashFiles('**/requirements*.txt', '**/pyproject.toml') }}
33+
restore-keys: |
34+
${{ runner.os }}-pip-
35+
36+
- name: Install dependencies
37+
run: |
38+
python -m pip install --upgrade pip
39+
pip install -e ".[dev]"
40+
# Install jsonschema for schema validation tests
41+
pip install jsonschema
42+
43+
- name: Run unit tests
44+
run: |
45+
pytest tests/unit/ -v \
46+
--cov=stagehand \
47+
--cov-report=xml \
48+
--cov-report=term-missing \
49+
--junit-xml=junit-unit-${{ matrix.python-version }}.xml \
50+
-m "unit and not slow"
51+
52+
- name: Upload unit test results
53+
uses: actions/upload-artifact@v3
54+
if: always()
55+
with:
56+
name: unit-test-results-${{ matrix.python-version }}
57+
path: junit-unit-${{ matrix.python-version }}.xml
58+
59+
- name: Upload coverage to Codecov
60+
uses: codecov/codecov-action@v3
61+
if: matrix.python-version == '3.11'
62+
with:
63+
file: ./coverage.xml
64+
flags: unit
65+
name: unit-tests
66+
67+
test-integration:
68+
name: Integration Tests
69+
runs-on: ubuntu-latest
70+
needs: test-unit
71+
strategy:
72+
matrix:
73+
test-category: ["local", "mock", "e2e"]
74+
75+
steps:
76+
- uses: actions/checkout@v4
77+
78+
- name: Set up Python 3.11
79+
uses: actions/setup-python@v4
80+
with:
81+
python-version: "3.11"
82+
83+
- name: Install dependencies
84+
run: |
85+
python -m pip install --upgrade pip
86+
pip install -e ".[dev]"
87+
pip install jsonschema
88+
# Install Playwright browsers for integration tests
89+
playwright install chromium
90+
91+
- name: Run integration tests - ${{ matrix.test-category }}
92+
run: |
93+
pytest tests/integration/ -v \
94+
--cov=stagehand \
95+
--cov-report=xml \
96+
--junit-xml=junit-integration-${{ matrix.test-category }}.xml \
97+
-m "${{ matrix.test-category }}"
98+
env:
99+
# Mock environment variables for testing
100+
BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY || 'mock-api-key' }}
101+
BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID || 'mock-project-id' }}
102+
MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
103+
STAGEHAND_API_URL: "http://localhost:3000"
104+
105+
- name: Upload integration test results
106+
uses: actions/upload-artifact@v3
107+
if: always()
108+
with:
109+
name: integration-test-results-${{ matrix.test-category }}
110+
path: junit-integration-${{ matrix.test-category }}.xml
111+
112+
test-browserbase:
113+
name: Browserbase Integration Tests
114+
runs-on: ubuntu-latest
115+
needs: test-unit
116+
if: github.event_name == 'schedule' || contains(github.event.head_commit.message, '[test-browserbase]')
117+
118+
steps:
119+
- uses: actions/checkout@v4
120+
121+
- name: Set up Python 3.11
122+
uses: actions/setup-python@v4
123+
with:
124+
python-version: "3.11"
125+
126+
- name: Install dependencies
127+
run: |
128+
python -m pip install --upgrade pip
129+
pip install -e ".[dev]"
130+
pip install jsonschema
131+
132+
- name: Run Browserbase tests
133+
run: |
134+
pytest tests/ -v \
135+
--cov=stagehand \
136+
--cov-report=xml \
137+
--junit-xml=junit-browserbase.xml \
138+
-m "browserbase" \
139+
--tb=short
140+
env:
141+
BROWSERBASE_API_KEY: ${{ secrets.BROWSERBASE_API_KEY }}
142+
BROWSERBASE_PROJECT_ID: ${{ secrets.BROWSERBASE_PROJECT_ID }}
143+
MODEL_API_KEY: ${{ secrets.MODEL_API_KEY }}
144+
STAGEHAND_API_URL: ${{ secrets.STAGEHAND_API_URL }}
145+
146+
- name: Upload Browserbase test results
147+
uses: actions/upload-artifact@v3
148+
if: always()
149+
with:
150+
name: browserbase-test-results
151+
path: junit-browserbase.xml
152+
153+
test-performance:
154+
name: Performance Tests
155+
runs-on: ubuntu-latest
156+
needs: test-unit
157+
if: github.event_name == 'schedule' || contains(github.event.head_commit.message, '[test-performance]')
158+
159+
steps:
160+
- uses: actions/checkout@v4
161+
162+
- name: Set up Python 3.11
163+
uses: actions/setup-python@v4
164+
with:
165+
python-version: "3.11"
166+
167+
- name: Install dependencies
168+
run: |
169+
python -m pip install --upgrade pip
170+
pip install -e ".[dev]"
171+
pip install jsonschema
172+
playwright install chromium
173+
174+
- name: Run performance tests
175+
run: |
176+
pytest tests/performance/ -v \
177+
--junit-xml=junit-performance.xml \
178+
-m "performance" \
179+
--tb=short
180+
env:
181+
MODEL_API_KEY: ${{ secrets.MODEL_API_KEY || 'mock-model-key' }}
182+
183+
- name: Upload performance test results
184+
uses: actions/upload-artifact@v3
185+
if: always()
186+
with:
187+
name: performance-test-results
188+
path: junit-performance.xml
189+
190+
smoke-tests:
191+
name: Smoke Tests
192+
runs-on: ubuntu-latest
193+
194+
steps:
195+
- uses: actions/checkout@v4
196+
197+
- name: Set up Python 3.11
198+
uses: actions/setup-python@v4
199+
with:
200+
python-version: "3.11"
201+
202+
- name: Install dependencies
203+
run: |
204+
python -m pip install --upgrade pip
205+
pip install -e ".[dev]"
206+
pip install jsonschema
207+
208+
- name: Run smoke tests
209+
run: |
210+
pytest tests/ -v \
211+
--junit-xml=junit-smoke.xml \
212+
-m "smoke" \
213+
--tb=line \
214+
--maxfail=5
215+
216+
- name: Upload smoke test results
217+
uses: actions/upload-artifact@v3
218+
if: always()
219+
with:
220+
name: smoke-test-results
221+
path: junit-smoke.xml
222+
223+
lint-and-format:
224+
name: Linting and Formatting
225+
runs-on: ubuntu-latest
226+
227+
steps:
228+
- uses: actions/checkout@v4
229+
230+
- name: Set up Python 3.11
231+
uses: actions/setup-python@v4
232+
with:
233+
python-version: "3.11"
234+
235+
- name: Install dependencies
236+
run: |
237+
python -m pip install --upgrade pip
238+
pip install -e ".[dev]"
239+
240+
- name: Run ruff linting
241+
run: |
242+
ruff check stagehand/ tests/ --output-format=github
243+
244+
- name: Run ruff formatting check
245+
run: |
246+
ruff format --check stagehand/ tests/
247+
248+
- name: Run mypy type checking
249+
run: |
250+
mypy stagehand/ --ignore-missing-imports
251+
252+
- name: Check import sorting
253+
run: |
254+
isort --check-only stagehand/ tests/
255+
256+
coverage-report:
257+
name: Coverage Report
258+
runs-on: ubuntu-latest
259+
needs: [test-unit, test-integration]
260+
if: always()
261+
262+
steps:
263+
- uses: actions/checkout@v4
264+
265+
- name: Set up Python 3.11
266+
uses: actions/setup-python@v4
267+
with:
268+
python-version: "3.11"
269+
270+
- name: Install dependencies
271+
run: |
272+
python -m pip install --upgrade pip
273+
pip install coverage[toml] codecov
274+
275+
- name: Download coverage artifacts
276+
uses: actions/download-artifact@v3
277+
with:
278+
path: coverage-reports/
279+
280+
- name: Combine coverage reports
281+
run: |
282+
coverage combine coverage-reports/**/.coverage*
283+
coverage report --show-missing
284+
coverage html
285+
coverage xml
286+
287+
- name: Upload combined coverage
288+
uses: codecov/codecov-action@v3
289+
with:
290+
file: ./coverage.xml
291+
name: combined-coverage
292+
293+
- name: Upload coverage HTML report
294+
uses: actions/upload-artifact@v3
295+
with:
296+
name: coverage-html-report
297+
path: htmlcov/
298+
299+
test-summary:
300+
name: Test Summary
301+
runs-on: ubuntu-latest
302+
needs: [test-unit, test-integration, smoke-tests, lint-and-format]
303+
if: always()
304+
305+
steps:
306+
- name: Download all test artifacts
307+
uses: actions/download-artifact@v3
308+
with:
309+
path: test-results/
310+
311+
- name: Generate test summary
312+
run: |
313+
echo "## Test Results Summary" >> $GITHUB_STEP_SUMMARY
314+
echo "" >> $GITHUB_STEP_SUMMARY
315+
316+
# Count test files
317+
UNIT_TESTS=$(find test-results/ -name "junit-unit-*.xml" | wc -l)
318+
INTEGRATION_TESTS=$(find test-results/ -name "junit-integration-*.xml" | wc -l)
319+
320+
echo "- Unit test configurations: $UNIT_TESTS" >> $GITHUB_STEP_SUMMARY
321+
echo "- Integration test categories: $INTEGRATION_TESTS" >> $GITHUB_STEP_SUMMARY
322+
323+
# Check for test failures
324+
if [ -f test-results/*/junit-*.xml ]; then
325+
echo "- Test artifacts generated successfully ✅" >> $GITHUB_STEP_SUMMARY
326+
else
327+
echo "- Test artifacts missing ❌" >> $GITHUB_STEP_SUMMARY
328+
fi
329+
330+
echo "" >> $GITHUB_STEP_SUMMARY
331+
echo "Detailed results are available in the artifacts section." >> $GITHUB_STEP_SUMMARY

0 commit comments

Comments
 (0)