Skip to content

Commit a23545a

Browse files
tylerslatontylerslaton
authored
ci: more granular test filtering and better testing structure (#602)
Signed-off-by: Tyler Slaton <[email protected]>
1 parent ce6922d commit a23545a

File tree

9 files changed

+457
-356
lines changed

9 files changed

+457
-356
lines changed

‎.github/workflows/check-generated-files.yml‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1,14 +1,21 @@
1-
name: Check Generated Files
1+
name: static
22

33
on:
44
push:
55
branches: [main]
6+
paths:
7+
- "apps/dojo/**"
8+
- "integrations/**"
9+
- ".github/workflows/check-generated-files.yml"
610
pull_request:
711
branches: [main]
12+
paths:
13+
- "apps/dojo/**"
14+
- "integrations/**"
15+
- ".github/workflows/check-generated-files.yml"
816

917
jobs:
10-
check-files-json:
11-
name: Check files.json
18+
check-dojo-generated-files:
1219
runs-on: ubuntu-latest
1320
steps:
1421
- name: Checkout code
@@ -17,7 +24,7 @@ jobs:
1724
- name: Set up Node.js
1825
uses: actions/setup-node@v4
1926
with:
20-
node-version: '22'
27+
node-version: "22"
2128

2229
- name: Install pnpm
2330
uses: pnpm/action-setup@v4

‎.github/workflows/dojo-e2e.yml‎

Lines changed: 132 additions & 118 deletions
Original file line numberDiff line numberDiff line change
@@ -3,52 +3,66 @@ name: e2e
33
on:
44
push:
55
branches: [main]
6+
paths:
7+
- "integrations/**"
8+
- "apps/**"
9+
- "middlewares/**"
10+
- "pnpm-lock.yaml"
11+
- "pnpm-workspace.yaml"
12+
- ".github/workflows/dojo-e2e.yml"
613
pull_request:
714
branches: [main]
15+
paths:
16+
- "integrations/**"
17+
- "apps/**"
18+
- "middlewares/**"
19+
- "pnpm-lock.yaml"
20+
- "pnpm-workspace.yaml"
21+
- ".github/workflows/dojo-e2e.yml"
822

923
jobs:
10-
e2e:
11-
name: ${{ matrix.suite }}
24+
dojo:
25+
name: dojo / ${{ matrix.suite }}
1226
runs-on: depot-ubuntu-24.04
1327
strategy:
1428
fail-fast: false
1529
matrix:
1630
include:
1731
- suite: a2a-middleware
1832
test_path: tests/a2aMiddlewareTests
19-
services: ["dojo","a2a-middleware"]
33+
services: ["dojo", "a2a-middleware"]
2034
wait_on: http://localhost:9999,tcp:localhost:8011,tcp:localhost:8012,tcp:localhost:8013,tcp:localhost:8014
2135
- suite: adk-middleware
2236
test_path: tests/adkMiddlewareTests
23-
services: ["dojo","adk-middleware"]
37+
services: ["dojo", "adk-middleware"]
2438
wait_on: http://localhost:9999,tcp:localhost:8010
2539
- suite: agno
2640
test_path: tests/agnoTests
27-
services: ["dojo","agno"]
41+
services: ["dojo", "agno"]
2842
wait_on: http://localhost:9999,tcp:localhost:8002
2943
- suite: crew-ai
3044
test_path: tests/crewAITests
31-
services: ["dojo","crew-ai"]
45+
services: ["dojo", "crew-ai"]
3246
wait_on: http://localhost:9999,tcp:localhost:8003
3347
- suite: langgraph-python
3448
test_path: tests/langgraphPythonTests
35-
services: ["dojo","langgraph-platform-python"]
49+
services: ["dojo", "langgraph-platform-python"]
3650
wait_on: http://localhost:9999,tcp:localhost:8005
3751
- suite: langgraph-typescript
3852
test_path: tests/langgraphTypescriptTests
39-
services: ["dojo","langgraph-platform-typescript"]
53+
services: ["dojo", "langgraph-platform-typescript"]
4054
wait_on: http://localhost:9999,tcp:localhost:8006
4155
- suite: langgraph-fastapi
4256
test_path: tests/langgraphFastAPITests
43-
services: ["dojo","langgraph-fastapi"]
57+
services: ["dojo", "langgraph-fastapi"]
4458
wait_on: http://localhost:9999,tcp:localhost:8004
4559
- suite: llama-index
4660
test_path: tests/llamaIndexTests
47-
services: ["dojo","llama-index"]
61+
services: ["dojo", "llama-index"]
4862
wait_on: http://localhost:9999,tcp:localhost:8007
4963
- suite: mastra
5064
test_path: tests/mastraTests
51-
services: ["dojo","mastra"]
65+
services: ["dojo", "mastra"]
5266
wait_on: http://localhost:9999,tcp:localhost:8008
5367
- suite: mastra-agent-local
5468
test_path: tests/mastraAgentLocalTests
@@ -60,125 +74,125 @@ jobs:
6074
wait_on: http://localhost:9999
6175
- suite: pydantic-ai
6276
test_path: tests/pydanticAITests
63-
services: ["dojo","pydantic-ai"]
77+
services: ["dojo", "pydantic-ai"]
6478
wait_on: http://localhost:9999,tcp:localhost:8009
6579
- suite: server-starter
6680
test_path: tests/serverStarterTests
67-
services: ["dojo","server-starter"]
81+
services: ["dojo", "server-starter"]
6882
wait_on: http://localhost:9999,tcp:localhost:8000
6983
- suite: server-starter-all
7084
test_path: tests/serverStarterAllFeaturesTests
71-
services: ["dojo","server-starter-all"]
85+
services: ["dojo", "server-starter-all"]
7286
wait_on: http://localhost:9999,tcp:localhost:8001
7387
# - suite: vercel-ai-sdk
7488
# test_path: tests/vercelAISdkTests
7589
# services: ["dojo"]
7690
# wait_on: http://localhost:9999
7791

7892
steps:
79-
- name: Checkout code
80-
uses: actions/checkout@v4
81-
82-
- name: Set up Node.js
83-
uses: actions/setup-node@v4
84-
with:
85-
node-version: '22'
86-
87-
- name: Install pnpm
88-
uses: pnpm/action-setup@v4
89-
with:
90-
version: 10.13.1
91-
92-
# Now that pnpm is available, cache its store to speed installs
93-
- name: Resolve pnpm store path
94-
id: pnpm-store
95-
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
96-
97-
- name: Cache pnpm store
98-
uses: actions/cache@v4
99-
with:
100-
path: ${{ env.STORE_PATH }}
101-
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
102-
restore-keys: |
103-
${{ runner.os }}-pnpm-store-
104-
105-
# Cache Python tool caches and virtualenvs; restore only to avoid long saves
106-
- name: Cache Python dependencies (restore-only)
107-
id: cache-python
108-
uses: actions/cache/restore@v4
109-
with:
110-
path: |
111-
~/.cache/pip
112-
~/.cache/pypoetry
113-
~/.cache/uv
114-
**/.venv
115-
key: ${{ runner.os }}-pydeps-${{ hashFiles('**/poetry.lock', '**/pyproject.toml') }}
116-
restore-keys: |
117-
${{ runner.os }}-pydeps-
118-
119-
- name: Install Poetry
120-
uses: snok/install-poetry@v1
121-
with:
122-
version: latest
123-
virtualenvs-create: true
124-
virtualenvs-in-project: true
125-
126-
- name: Install uv
127-
uses: astral-sh/setup-uv@v6
128-
129-
- name: Install dependencies
130-
run: pnpm install --frozen-lockfile
131-
132-
- name: Prepare dojo for e2e
133-
working-directory: apps/dojo
134-
if: ${{ join(matrix.services, ',') != '' }}
135-
run: node ./scripts/prep-dojo-everything.js --only ${{ join(matrix.services, ',') }}
136-
137-
- name: Install e2e dependencies
138-
working-directory: apps/dojo/e2e
139-
run: |
140-
pnpm install
141-
142-
- name: write langgraph env files
143-
working-directory: integrations/langgraph
144-
env:
145-
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
146-
LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
147-
if: ${{ contains(join(matrix.services, ','), 'langgraph-fastapi') || contains(join(matrix.services, ','), 'langgraph-platform-python') || contains(join(matrix.services, ','), 'langgraph-platform-typescript') }}
148-
run: |
149-
echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > python/examples/.env
150-
echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> python/examples/.env
151-
echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > typescript/examples/.env
152-
echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> typescript/examples/.env
153-
154-
- name: Run dojo+agents
155-
uses: JarvusInnovations/background-action@v1
156-
env:
157-
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
158-
LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
159-
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
160-
if: ${{ join(matrix.services, ',') != '' && contains(join(matrix.services, ','), 'dojo') }}
161-
with:
93+
- name: Checkout code
94+
uses: actions/checkout@v4
95+
96+
- name: Set up Node.js
97+
uses: actions/setup-node@v4
98+
with:
99+
node-version: "22"
100+
101+
- name: Install pnpm
102+
uses: pnpm/action-setup@v4
103+
with:
104+
version: 10.13.1
105+
106+
# Now that pnpm is available, cache its store to speed installs
107+
- name: Resolve pnpm store path
108+
id: pnpm-store
109+
run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
110+
111+
- name: Cache pnpm store
112+
uses: actions/cache@v4
113+
with:
114+
path: ${{ env.STORE_PATH }}
115+
key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
116+
restore-keys: |
117+
${{ runner.os }}-pnpm-store-
118+
119+
# Cache Python tool caches and virtualenvs; restore only to avoid long saves
120+
- name: Cache Python dependencies (restore-only)
121+
id: cache-python
122+
uses: actions/cache/restore@v4
123+
with:
124+
path: |
125+
~/.cache/pip
126+
~/.cache/pypoetry
127+
~/.cache/uv
128+
**/.venv
129+
key: ${{ runner.os }}-pydeps-${{ hashFiles('**/poetry.lock', '**/pyproject.toml') }}
130+
restore-keys: |
131+
${{ runner.os }}-pydeps-
132+
133+
- name: Install Poetry
134+
uses: snok/install-poetry@v1
135+
with:
136+
version: latest
137+
virtualenvs-create: true
138+
virtualenvs-in-project: true
139+
140+
- name: Install uv
141+
uses: astral-sh/setup-uv@v6
142+
143+
- name: Install dependencies
144+
run: pnpm install --frozen-lockfile
145+
146+
- name: Prepare dojo for e2e
147+
working-directory: apps/dojo
148+
if: ${{ join(matrix.services, ',') != '' }}
149+
run: node ./scripts/prep-dojo-everything.js --only ${{ join(matrix.services, ',') }}
150+
151+
- name: Install e2e dependencies
152+
working-directory: apps/dojo/e2e
153+
run: |
154+
pnpm install
155+
156+
- name: write langgraph env files
157+
working-directory: integrations/langgraph
158+
env:
159+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
160+
LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
161+
if: ${{ contains(join(matrix.services, ','), 'langgraph-fastapi') || contains(join(matrix.services, ','), 'langgraph-platform-python') || contains(join(matrix.services, ','), 'langgraph-platform-typescript') }}
162162
run: |
163-
node ../scripts/run-dojo-everything.js --only ${{ join(matrix.services, ',') }}
163+
echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > python/examples/.env
164+
echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> python/examples/.env
165+
echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > typescript/examples/.env
166+
echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> typescript/examples/.env
167+
168+
- name: Run dojo+agents
169+
uses: JarvusInnovations/background-action@v1
170+
env:
171+
OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
172+
LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
173+
GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
174+
if: ${{ join(matrix.services, ',') != '' && contains(join(matrix.services, ','), 'dojo') }}
175+
with:
176+
run: |
177+
node ../scripts/run-dojo-everything.js --only ${{ join(matrix.services, ',') }}
178+
working-directory: apps/dojo/e2e
179+
wait-on: ${{ matrix.wait_on }}
180+
wait-for: 300000
181+
182+
- name: Run tests – ${{ matrix.suite }}
164183
working-directory: apps/dojo/e2e
165-
wait-on: ${{ matrix.wait_on }}
166-
wait-for: 300000
167-
168-
- name: Run tests – ${{ matrix.suite }}
169-
working-directory: apps/dojo/e2e
170-
env:
171-
BASE_URL: http://localhost:9999
172-
PLAYWRIGHT_SUITE: ${{ matrix.suite }}
173-
run: |
174-
pnpm test -- ${{ matrix.test_path }}
175-
176-
- name: Upload traces – ${{ matrix.suite }}
177-
if: always() # Uploads artifacts even if tests fail
178-
uses: actions/upload-artifact@v4
179-
with:
180-
name: ${{ matrix.suite }}-playwright-traces
181-
path: |
182-
apps/dojo/e2e/test-results/${{ matrix.suite }}/**/*
183-
apps/dojo/e2e/playwright-report/**/*
184-
retention-days: 7
184+
env:
185+
BASE_URL: http://localhost:9999
186+
PLAYWRIGHT_SUITE: ${{ matrix.suite }}
187+
run: |
188+
pnpm test -- ${{ matrix.test_path }}
189+
190+
- name: Upload traces – ${{ matrix.suite }}
191+
if: always() # Uploads artifacts even if tests fail
192+
uses: actions/upload-artifact@v4
193+
with:
194+
name: ${{ matrix.suite }}-playwright-traces
195+
path: |
196+
apps/dojo/e2e/test-results/${{ matrix.suite }}/**/*
197+
apps/dojo/e2e/playwright-report/**/*
198+
retention-days: 7

0 commit comments

Comments
 (0)