Skip to content

Commit b88c39e

Browse files
tylerslatonjohnprg
tylerslaton
authored and
johnprg
committed
ci: simplify CI logic for determining what frameworks to run in e2e tests on
Signed-off-by: Tyler Slaton <[email protected]>
1 parent a8ed6ec commit b88c39e

File tree

1 file changed

+84
-162
lines changed

1 file changed

+84
-162
lines changed

‎.github/workflows/dojo-e2e.yml‎

Lines changed: 84 additions & 162 deletions
Original file line numberDiff line numberDiff line change
@@ -11,208 +11,130 @@ jobs:
1111
name: Determine suites to run
1212
runs-on: ubuntu-latest
1313
outputs:
14-
matrix: ${{ steps.set-matrix.outputs.matrix }}
15-
should_run: ${{ steps.set-matrix.outputs.should_run }}
14+
suites: ${{ steps.set-suites.outputs.suites }}
15+
should_run: ${{ steps.set-suites.outputs.should_run }}
16+
matrix: ${{ steps.set-suites.outputs.matrix }}
1617
steps:
1718
- name: Checkout code
1819
uses: actions/checkout@v4
1920
with:
2021
fetch-depth: 0
2122

22-
- name: Detect changed areas
23+
- name: Detect changes
2324
id: filter
2425
uses: dorny/paths-filter@v3
2526
with:
2627
filters: |
27-
core_ts:
28+
# Core changes that run ALL suites
29+
core_changes:
2830
- 'typescript-sdk/packages/**'
2931
- 'typescript-sdk/package.json'
30-
- 'typescript-sdk/pnpm-lock.yaml'
3132
- 'typescript-sdk/pnpm-workspace.yaml'
3233
- 'typescript-sdk/tsconfig.json'
3334
- 'typescript-sdk/turbo.json'
34-
core_py:
3535
- 'python-sdk/**'
36-
# Treat e2e config files as core, but tests are filtered per-suite below
37-
e2e_config:
3836
- 'typescript-sdk/apps/dojo/e2e/**'
3937
- '!typescript-sdk/apps/dojo/e2e/tests/**'
40-
e2e_scripts:
4138
- 'typescript-sdk/apps/dojo/scripts/**'
42-
# Per-suite e2e test filters so changing a test only runs that suite
43-
e2e_agno_tests:
39+
- '.github/workflows/dojo-e2e.yml'
40+
41+
# Individual suite changes (source OR tests)
42+
agno_changes:
43+
- 'typescript-sdk/integrations/agno/**'
4444
- 'typescript-sdk/apps/dojo/e2e/tests/agnoTests/**'
45-
e2e_crew_ai_tests:
45+
crew_ai_changes:
46+
- 'typescript-sdk/integrations/crewai/**'
4647
- 'typescript-sdk/apps/dojo/e2e/tests/crewAITests/**'
47-
e2e_langgraph_tests:
48+
langgraph_changes:
49+
- 'typescript-sdk/integrations/langgraph/**'
4850
- 'typescript-sdk/apps/dojo/e2e/tests/langgraphTests/**'
49-
e2e_langgraph_fastapi_tests:
5051
- 'typescript-sdk/apps/dojo/e2e/tests/langgraphFastAPITests/**'
51-
e2e_llama_index_tests:
52+
llama_index_changes:
53+
- 'typescript-sdk/integrations/llamaindex/**'
5254
- 'typescript-sdk/apps/dojo/e2e/tests/llamaIndexTests/**'
53-
e2e_mastra_tests:
55+
mastra_changes:
56+
- 'typescript-sdk/integrations/mastra/**'
5457
- 'typescript-sdk/apps/dojo/e2e/tests/mastraTests/**'
55-
e2e_mastra_agent_local_tests:
5658
- 'typescript-sdk/apps/dojo/e2e/tests/mastraAgentLocalTests/**'
57-
e2e_middleware_starter_tests:
58-
- 'typescript-sdk/apps/dojo/e2e/tests/middlewareStarterTests/**'
59-
e2e_pydantic_ai_tests:
60-
- 'typescript-sdk/apps/dojo/e2e/tests/pydanticAITests/**'
61-
e2e_server_starter_tests:
62-
- 'typescript-sdk/apps/dojo/e2e/tests/serverStarterTests/**'
63-
e2e_server_starter_all_tests:
64-
- 'typescript-sdk/apps/dojo/e2e/tests/serverStarterAllFeaturesTests/**'
65-
e2e_vercel_ai_sdk_tests:
66-
- 'typescript-sdk/apps/dojo/e2e/tests/vercelAISdkTests/**'
67-
workflow_self:
68-
- '.github/workflows/dojo-e2e.yml'
69-
agno:
70-
- 'typescript-sdk/integrations/agno/**'
71-
crew_ai:
72-
- 'typescript-sdk/integrations/crewai/**'
73-
langgraph:
74-
- 'typescript-sdk/integrations/langgraph/**'
75-
llama_index:
76-
- 'typescript-sdk/integrations/llamaindex/**'
77-
mastra:
78-
- 'typescript-sdk/integrations/mastra/**'
79-
middleware_starter:
59+
middleware_starter_changes:
8060
- 'typescript-sdk/integrations/middleware-starter/**'
81-
pydantic_ai:
61+
- 'typescript-sdk/apps/dojo/e2e/tests/middlewareStarterTests/**'
62+
pydantic_ai_changes:
8263
- 'typescript-sdk/integrations/pydantic-ai/**'
83-
server_starter:
64+
- 'typescript-sdk/apps/dojo/e2e/tests/pydanticAITests/**'
65+
server_starter_changes:
8466
- 'typescript-sdk/integrations/server-starter/**'
85-
server_starter_all:
67+
- 'typescript-sdk/apps/dojo/e2e/tests/serverStarterTests/**'
68+
server_starter_all_changes:
8669
- 'typescript-sdk/integrations/server-starter-all-features/**'
87-
vercel_ai_sdk:
70+
- 'typescript-sdk/apps/dojo/e2e/tests/serverStarterAllFeaturesTests/**'
71+
vercel_ai_sdk_changes:
8872
- 'typescript-sdk/integrations/vercel-ai-sdk/**'
73+
- 'typescript-sdk/apps/dojo/e2e/tests/vercelAISdkTests/**'
8974
90-
- name: Build dynamic matrix
91-
id: set-matrix
92-
env:
93-
CORE_TS: ${{ steps.filter.outputs.core_ts }}
94-
CORE_PY: ${{ steps.filter.outputs.core_py }}
95-
E2E_CONFIG: ${{ steps.filter.outputs.e2e_config }}
96-
E2E_SCRIPTS: ${{ steps.filter.outputs.e2e_scripts }}
97-
E2E_AGNO_TESTS: ${{ steps.filter.outputs.e2e_agno_tests }}
98-
E2E_CREW_AI_TESTS: ${{ steps.filter.outputs.e2e_crew_ai_tests }}
99-
E2E_LANGGRAPH_TESTS: ${{ steps.filter.outputs.e2e_langgraph_tests }}
100-
E2E_LANGGRAPH_FASTAPI_TESTS: ${{ steps.filter.outputs.e2e_langgraph_fastapi_tests }}
101-
E2E_LLAMA_INDEX_TESTS: ${{ steps.filter.outputs.e2e_llama_index_tests }}
102-
E2E_MASTRA_TESTS: ${{ steps.filter.outputs.e2e_mastra_tests }}
103-
E2E_MASTRA_AGENT_LOCAL_TESTS: ${{ steps.filter.outputs.e2e_mastra_agent_local_tests }}
104-
E2E_MIDDLEWARE_STARTER_TESTS: ${{ steps.filter.outputs.e2e_middleware_starter_tests }}
105-
E2E_PYDANTIC_AI_TESTS: ${{ steps.filter.outputs.e2e_pydantic_ai_tests }}
106-
E2E_SERVER_STARTER_TESTS: ${{ steps.filter.outputs.e2e_server_starter_tests }}
107-
E2E_SERVER_STARTER_ALL_TESTS: ${{ steps.filter.outputs.e2e_server_starter_all_tests }}
108-
E2E_VERCEL_AI_SDK_TESTS: ${{ steps.filter.outputs.e2e_vercel_ai_sdk_tests }}
109-
WORKFLOW_SELF: ${{ steps.filter.outputs.workflow_self }}
110-
AGNO: ${{ steps.filter.outputs.agno }}
111-
CREW_AI: ${{ steps.filter.outputs.crew_ai }}
112-
LANGGRAPH: ${{ steps.filter.outputs.langgraph }}
113-
LLAMA_INDEX: ${{ steps.filter.outputs.llama_index }}
114-
MASTRA: ${{ steps.filter.outputs.mastra }}
115-
MIDDLEWARE_STARTER: ${{ steps.filter.outputs.middleware_starter }}
116-
PYDANTIC_AI: ${{ steps.filter.outputs.pydantic_ai }}
117-
SERVER_STARTER: ${{ steps.filter.outputs.server_starter }}
118-
SERVER_STARTER_ALL: ${{ steps.filter.outputs.server_starter_all }}
119-
VERCEL_AI_SDK: ${{ steps.filter.outputs.vercel_ai_sdk }}
75+
- name: Set suites output
76+
id: set-suites
12077
run: |
121-
python3 - << 'PY'
122-
import os, json
123-
124-
all_entries = [
125-
{"suite": "agno", "test_path": "tests/agnoTests", "services": ["dojo","agno"], "wait_on": "http://localhost:9999,tcp:localhost:8002"},
126-
{"suite": "crew-ai", "test_path": "tests/crewAITests", "services": ["dojo","crew-ai"], "wait_on": "http://localhost:9999,tcp:localhost:8003"},
127-
{"suite": "langgraph", "test_path": "tests/langgraphTests", "services": ["dojo","langgraph-platform-python","langgraph-platform-typescript"], "wait_on": "http://localhost:9999,tcp:localhost:8005,tcp:localhost:8006"},
128-
{"suite": "langgraph-fastapi", "test_path": "tests/langgraphFastAPITests", "services": ["dojo","langgraph-fastapi"], "wait_on": "http://localhost:9999,tcp:localhost:8004"},
129-
{"suite": "llama-index", "test_path": "tests/llamaIndexTests", "services": ["dojo","llama-index"], "wait_on": "http://localhost:9999,tcp:localhost:8007"},
130-
{"suite": "mastra", "test_path": "tests/mastraTests", "services": ["dojo","mastra"], "wait_on": "http://localhost:9999,tcp:localhost:8008"},
131-
{"suite": "mastra-agent-local", "test_path": "tests/mastraAgentLocalTests", "services": ["dojo"], "wait_on": "http://localhost:9999"},
132-
{"suite": "middleware-starter", "test_path": "tests/middlewareStarterTests", "services": ["dojo"], "wait_on": "http://localhost:9999"},
133-
{"suite": "pydantic-ai", "test_path": "tests/pydanticAITests", "services": ["dojo","pydantic-ai"], "wait_on": "http://localhost:9999,tcp:localhost:8009"},
134-
{"suite": "server-starter", "test_path": "tests/serverStarterTests", "services": ["dojo","server-starter"], "wait_on": "http://localhost:9999,tcp:localhost:8000"},
135-
{"suite": "server-starter-all", "test_path": "tests/serverStarterAllFeaturesTests", "services": ["dojo","server-starter-all"], "wait_on": "http://localhost:9999,tcp:localhost:8001"},
136-
{"suite": "vercel-ai-sdk", "test_path": "tests/vercelAISdkTests", "services": ["dojo"], "wait_on": "http://localhost:9999"},
137-
]
138-
139-
entry_by_suite = {e["suite"]: e for e in all_entries}
140-
core_changed = (
141-
(os.environ.get('CORE_TS') == 'true') or
142-
(os.environ.get('CORE_PY') == 'true') or
143-
(os.environ.get('E2E_CONFIG') == 'true') or
144-
(os.environ.get('E2E_SCRIPTS') == 'true') or
145-
(os.environ.get('WORKFLOW_SELF') == 'true')
146-
)
147-
148-
include = []
149-
if core_changed:
150-
include = all_entries
151-
else:
152-
if os.environ.get('AGNO') == 'true':
153-
include.append(entry_by_suite['agno'])
154-
if os.environ.get('CREW_AI') == 'true':
155-
include.append(entry_by_suite['crew-ai'])
156-
if os.environ.get('LANGGRAPH') == 'true':
157-
include.append(entry_by_suite['langgraph'])
158-
include.append(entry_by_suite['langgraph-fastapi'])
159-
if os.environ.get('LLAMA_INDEX') == 'true':
160-
include.append(entry_by_suite['llama-index'])
161-
if os.environ.get('MASTRA') == 'true':
162-
include.append(entry_by_suite['mastra'])
163-
include.append(entry_by_suite['mastra-agent-local'])
164-
if os.environ.get('MIDDLEWARE_STARTER') == 'true':
165-
include.append(entry_by_suite['middleware-starter'])
166-
if os.environ.get('PYDANTIC_AI') == 'true':
167-
include.append(entry_by_suite['pydantic-ai'])
168-
if os.environ.get('SERVER_STARTER') == 'true':
169-
include.append(entry_by_suite['server-starter'])
170-
if os.environ.get('SERVER_STARTER_ALL') == 'true':
171-
include.append(entry_by_suite['server-starter-all'])
172-
if os.environ.get('VERCEL_AI_SDK') == 'true':
173-
include.append(entry_by_suite['vercel-ai-sdk'])
174-
175-
# Include suites whose tests changed directly
176-
if os.environ.get('E2E_AGNO_TESTS') == 'true':
177-
include.append(entry_by_suite['agno'])
178-
if os.environ.get('E2E_CREW_AI_TESTS') == 'true':
179-
include.append(entry_by_suite['crew-ai'])
180-
if os.environ.get('E2E_LANGGRAPH_TESTS') == 'true':
181-
include.append(entry_by_suite['langgraph'])
182-
if os.environ.get('E2E_LANGGRAPH_FASTAPI_TESTS') == 'true':
183-
include.append(entry_by_suite['langgraph-fastapi'])
184-
if os.environ.get('E2E_LLAMA_INDEX_TESTS') == 'true':
185-
include.append(entry_by_suite['llama-index'])
186-
if os.environ.get('E2E_MASTRA_TESTS') == 'true':
187-
include.append(entry_by_suite['mastra'])
188-
if os.environ.get('E2E_MASTRA_AGENT_LOCAL_TESTS') == 'true':
189-
include.append(entry_by_suite['mastra-agent-local'])
190-
if os.environ.get('E2E_MIDDLEWARE_STARTER_TESTS') == 'true':
191-
include.append(entry_by_suite['middleware-starter'])
192-
if os.environ.get('E2E_PYDANTIC_AI_TESTS') == 'true':
193-
include.append(entry_by_suite['pydantic-ai'])
194-
if os.environ.get('E2E_SERVER_STARTER_TESTS') == 'true':
195-
include.append(entry_by_suite['server-starter'])
196-
if os.environ.get('E2E_SERVER_STARTER_ALL_TESTS') == 'true':
197-
include.append(entry_by_suite['server-starter-all'])
198-
if os.environ.get('E2E_VERCEL_AI_SDK_TESTS') == 'true':
199-
include.append(entry_by_suite['vercel-ai-sdk'])
200-
201-
# de-duplicate by suite in case multiple reasons include the same suite
202-
include_unique = list({e['suite']: e for e in include}.values())
203-
matrix = {"include": include_unique}
204-
with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
205-
fh.write(f"matrix={json.dumps(matrix)}\n")
206-
fh.write(f"should_run={'true' if include else 'false'}\n")
207-
PY
78+
set -euo pipefail
79+
80+
# Define all possible matrix configurations
81+
ALL_MATRIX='[
82+
{"suite":"agno","test_path":"tests/agnoTests","services":["dojo","agno"],"wait_on":"http://localhost:9999,tcp:localhost:8002"},
83+
{"suite":"crew-ai","test_path":"tests/crewAITests","services":["dojo","crew-ai"],"wait_on":"http://localhost:9999,tcp:localhost:8003"},
84+
{"suite":"langgraph","test_path":"tests/langgraphTests","services":["dojo","langgraph-platform-python","langgraph-platform-typescript"],"wait_on":"http://localhost:9999,tcp:localhost:8005,tcp:localhost:8006"},
85+
{"suite":"langgraph-fastapi","test_path":"tests/langgraphFastAPITests","services":["dojo","langgraph-fastapi"],"wait_on":"http://localhost:9999,tcp:localhost:8004"},
86+
{"suite":"llama-index","test_path":"tests/llamaIndexTests","services":["dojo","llama-index"],"wait_on":"http://localhost:9999,tcp:localhost:8007"},
87+
{"suite":"mastra","test_path":"tests/mastraTests","services":["dojo","mastra"],"wait_on":"http://localhost:9999,tcp:localhost:8008"},
88+
{"suite":"mastra-agent-local","test_path":"tests/mastraAgentLocalTests","services":["dojo"],"wait_on":"http://localhost:9999"},
89+
{"suite":"middleware-starter","test_path":"tests/middlewareStarterTests","services":["dojo"],"wait_on":"http://localhost:9999"},
90+
{"suite":"pydantic-ai","test_path":"tests/pydanticAITests","services":["dojo","pydantic-ai"],"wait_on":"http://localhost:9999,tcp:localhost:8009"},
91+
{"suite":"server-starter","test_path":"tests/serverStarterTests","services":["dojo","server-starter"],"wait_on":"http://localhost:9999,tcp:localhost:8000"},
92+
{"suite":"server-starter-all","test_path":"tests/serverStarterAllFeaturesTests","services":["dojo","server-starter-all"],"wait_on":"http://localhost:9999,tcp:localhost:8001"},
93+
{"suite":"vercel-ai-sdk","test_path":"tests/vercelAISdkTests","services":["dojo"],"wait_on":"http://localhost:9999"}
94+
]'
95+
96+
# If core changes, run all suites
97+
if [[ "${{ steps.filter.outputs.core_changes }}" == "true" ]]; then
98+
SUITES="agno,crew-ai,langgraph,langgraph-fastapi,llama-index,mastra,mastra-agent-local,middleware-starter,pydantic-ai,server-starter,server-starter-all,vercel-ai-sdk"
99+
else
100+
# Build list of changed suites
101+
SUITES=""
102+
[[ "${{ steps.filter.outputs.agno_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}agno"
103+
[[ "${{ steps.filter.outputs.crew_ai_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}crew-ai"
104+
[[ "${{ steps.filter.outputs.langgraph_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}langgraph,langgraph-fastapi"
105+
[[ "${{ steps.filter.outputs.llama_index_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}llama-index"
106+
[[ "${{ steps.filter.outputs.mastra_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}mastra,mastra-agent-local"
107+
[[ "${{ steps.filter.outputs.middleware_starter_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}middleware-starter"
108+
[[ "${{ steps.filter.outputs.pydantic_ai_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}pydantic-ai"
109+
[[ "${{ steps.filter.outputs.server_starter_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}server-starter"
110+
[[ "${{ steps.filter.outputs.server_starter_all_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}server-starter-all"
111+
[[ "${{ steps.filter.outputs.vercel_ai_sdk_changes }}" == "true" ]] && SUITES="${SUITES:+$SUITES,}vercel-ai-sdk"
112+
fi
113+
114+
echo "suites=${SUITES}" >> "$GITHUB_OUTPUT"
115+
116+
# Generate dynamic matrix based on changed suites
117+
if [[ -n "${SUITES}" ]]; then
118+
echo "should_run=true" >> "$GITHUB_OUTPUT"
119+
120+
# Filter matrix to only include changed suites
121+
FILTERED_MATRIX=$(echo "$ALL_MATRIX" | jq -c --arg suites "$SUITES" '
122+
[.[] | select(.suite as $suite | ($suites | split(",") | index($suite)))]
123+
')
124+
echo "matrix=${FILTERED_MATRIX}" >> "$GITHUB_OUTPUT"
125+
else
126+
echo "should_run=false" >> "$GITHUB_OUTPUT"
127+
echo "matrix=[]" >> "$GITHUB_OUTPUT"
128+
fi
208129
e2e:
209130
needs: changes
210131
if: ${{ needs.changes.outputs.should_run == 'true' }}
211132
name: ${{ matrix.suite }}
212133
runs-on: depot-ubuntu-24.04
213134
strategy:
214135
fail-fast: false
215-
matrix: ${{ fromJSON(needs.changes.outputs.matrix) }}
136+
matrix:
137+
include: ${{ fromJSON(needs.changes.outputs.matrix) }}
216138

217139
steps:
218140
- name: Checkout code

0 commit comments

Comments
 (0)