Skip to content

Commit ab1e2da

Browse files
tylerslatontylerslaton
committed
ci: simplify CI logic for determining what frameworks to run in e2e tests on
Signed-off-by: Tyler Slaton <[email protected]>
1 parent 3d461ca commit ab1e2da

File tree

1 file changed

+87
-155
lines changed

1 file changed

+87
-155
lines changed

‎.github/workflows/dojo-e2e.yml‎

Lines changed: 87 additions & 155 deletions
Original file line numberDiff line numberDiff line change
@@ -19,107 +19,18 @@ jobs:
1919
with:
2020
fetch-depth: 0
2121

22-
- name: Detect changed areas
23-
id: filter
24-
uses: dorny/paths-filter@v3
25-
with:
26-
filters: |
27-
core_ts:
28-
- 'typescript-sdk/packages/**'
29-
- 'typescript-sdk/package.json'
30-
- 'typescript-sdk/pnpm-lock.yaml'
31-
- 'typescript-sdk/pnpm-workspace.yaml'
32-
- 'typescript-sdk/tsconfig.json'
33-
- 'typescript-sdk/turbo.json'
34-
core_py:
35-
- 'python-sdk/**'
36-
# Treat e2e config files as core, but tests are filtered per-suite below
37-
e2e_config:
38-
- 'typescript-sdk/apps/dojo/e2e/**'
39-
- '!typescript-sdk/apps/dojo/e2e/tests/**'
40-
e2e_scripts:
41-
- 'typescript-sdk/apps/dojo/scripts/**'
42-
# Per-suite e2e test filters so changing a test only runs that suite
43-
e2e_agno_tests:
44-
- 'typescript-sdk/apps/dojo/e2e/tests/agnoTests/**'
45-
e2e_crew_ai_tests:
46-
- 'typescript-sdk/apps/dojo/e2e/tests/crewAITests/**'
47-
e2e_langgraph_tests:
48-
- 'typescript-sdk/apps/dojo/e2e/tests/langgraphTests/**'
49-
e2e_langgraph_fastapi_tests:
50-
- 'typescript-sdk/apps/dojo/e2e/tests/langgraphFastAPITests/**'
51-
e2e_llama_index_tests:
52-
- 'typescript-sdk/apps/dojo/e2e/tests/llamaIndexTests/**'
53-
e2e_mastra_tests:
54-
- 'typescript-sdk/apps/dojo/e2e/tests/mastraTests/**'
55-
e2e_mastra_agent_local_tests:
56-
- 'typescript-sdk/apps/dojo/e2e/tests/mastraAgentLocalTests/**'
57-
e2e_middleware_starter_tests:
58-
- 'typescript-sdk/apps/dojo/e2e/tests/middlewareStarterTests/**'
59-
e2e_pydantic_ai_tests:
60-
- 'typescript-sdk/apps/dojo/e2e/tests/pydanticAITests/**'
61-
e2e_server_starter_tests:
62-
- 'typescript-sdk/apps/dojo/e2e/tests/serverStarterTests/**'
63-
e2e_server_starter_all_tests:
64-
- 'typescript-sdk/apps/dojo/e2e/tests/serverStarterAllFeaturesTests/**'
65-
e2e_vercel_ai_sdk_tests:
66-
- 'typescript-sdk/apps/dojo/e2e/tests/vercelAISdkTests/**'
67-
workflow_self:
68-
- '.github/workflows/dojo-e2e.yml'
69-
agno:
70-
- 'typescript-sdk/integrations/agno/**'
71-
crew_ai:
72-
- 'typescript-sdk/integrations/crewai/**'
73-
langgraph:
74-
- 'typescript-sdk/integrations/langgraph/**'
75-
llama_index:
76-
- 'typescript-sdk/integrations/llamaindex/**'
77-
mastra:
78-
- 'typescript-sdk/integrations/mastra/**'
79-
middleware_starter:
80-
- 'typescript-sdk/integrations/middleware-starter/**'
81-
pydantic_ai:
82-
- 'typescript-sdk/integrations/pydantic-ai/**'
83-
server_starter:
84-
- 'typescript-sdk/integrations/server-starter/**'
85-
server_starter_all:
86-
- 'typescript-sdk/integrations/server-starter-all-features/**'
87-
vercel_ai_sdk:
88-
- 'typescript-sdk/integrations/vercel-ai-sdk/**'
22+
# Simplified: compute changed suites via git diff in the next step
8923

9024
- name: Build dynamic matrix
9125
id: set-matrix
9226
env:
93-
CORE_TS: ${{ steps.filter.outputs.core_ts }}
94-
CORE_PY: ${{ steps.filter.outputs.core_py }}
95-
E2E_CONFIG: ${{ steps.filter.outputs.e2e_config }}
96-
E2E_SCRIPTS: ${{ steps.filter.outputs.e2e_scripts }}
97-
E2E_AGNO_TESTS: ${{ steps.filter.outputs.e2e_agno_tests }}
98-
E2E_CREW_AI_TESTS: ${{ steps.filter.outputs.e2e_crew_ai_tests }}
99-
E2E_LANGGRAPH_TESTS: ${{ steps.filter.outputs.e2e_langgraph_tests }}
100-
E2E_LANGGRAPH_FASTAPI_TESTS: ${{ steps.filter.outputs.e2e_langgraph_fastapi_tests }}
101-
E2E_LLAMA_INDEX_TESTS: ${{ steps.filter.outputs.e2e_llama_index_tests }}
102-
E2E_MASTRA_TESTS: ${{ steps.filter.outputs.e2e_mastra_tests }}
103-
E2E_MASTRA_AGENT_LOCAL_TESTS: ${{ steps.filter.outputs.e2e_mastra_agent_local_tests }}
104-
E2E_MIDDLEWARE_STARTER_TESTS: ${{ steps.filter.outputs.e2e_middleware_starter_tests }}
105-
E2E_PYDANTIC_AI_TESTS: ${{ steps.filter.outputs.e2e_pydantic_ai_tests }}
106-
E2E_SERVER_STARTER_TESTS: ${{ steps.filter.outputs.e2e_server_starter_tests }}
107-
E2E_SERVER_STARTER_ALL_TESTS: ${{ steps.filter.outputs.e2e_server_starter_all_tests }}
108-
E2E_VERCEL_AI_SDK_TESTS: ${{ steps.filter.outputs.e2e_vercel_ai_sdk_tests }}
109-
WORKFLOW_SELF: ${{ steps.filter.outputs.workflow_self }}
110-
AGNO: ${{ steps.filter.outputs.agno }}
111-
CREW_AI: ${{ steps.filter.outputs.crew_ai }}
112-
LANGGRAPH: ${{ steps.filter.outputs.langgraph }}
113-
LLAMA_INDEX: ${{ steps.filter.outputs.llama_index }}
114-
MASTRA: ${{ steps.filter.outputs.mastra }}
115-
MIDDLEWARE_STARTER: ${{ steps.filter.outputs.middleware_starter }}
116-
PYDANTIC_AI: ${{ steps.filter.outputs.pydantic_ai }}
117-
SERVER_STARTER: ${{ steps.filter.outputs.server_starter }}
118-
SERVER_STARTER_ALL: ${{ steps.filter.outputs.server_starter_all }}
119-
VERCEL_AI_SDK: ${{ steps.filter.outputs.vercel_ai_sdk }}
27+
EVENT_NAME: ${{ github.event_name }}
28+
PR_BASE_SHA: ${{ github.event.pull_request.base.sha }}
29+
BEFORE_SHA: ${{ github.event.before }}
30+
HEAD_SHA: ${{ github.sha }}
12031
run: |
12132
python3 - << 'PY'
122-
import os, json
33+
import os, json, subprocess
12334
12435
all_entries = [
12536
{"suite": "agno", "test_path": "tests/agnoTests", "services": ["dojo","agno"], "wait_on": "http://localhost:9999,tcp:localhost:8002"},
@@ -137,73 +48,94 @@ jobs:
13748
]
13849
13950
entry_by_suite = {e["suite"]: e for e in all_entries}
140-
core_changed = (
141-
(os.environ.get('CORE_TS') == 'true') or
142-
(os.environ.get('CORE_PY') == 'true') or
143-
(os.environ.get('E2E_CONFIG') == 'true') or
144-
(os.environ.get('E2E_SCRIPTS') == 'true') or
145-
(os.environ.get('WORKFLOW_SELF') == 'true')
146-
)
14751
148-
include = []
52+
# Determine changed files between base and head
53+
event_name = os.getenv('EVENT_NAME')
54+
base = os.getenv('PR_BASE_SHA') if event_name == 'pull_request' else os.getenv('BEFORE_SHA')
55+
head = os.getenv('HEAD_SHA')
56+
57+
if not base:
58+
try:
59+
base = subprocess.check_output(['git', 'rev-parse', 'HEAD~1'], text=True).strip()
60+
except Exception:
61+
base = head
62+
63+
subprocess.run(['git', 'fetch', '--no-tags', '--prune', '--depth=0', 'origin'], check=False)
64+
diff = subprocess.run(['git', 'diff', '--name-only', f'{base}...{head}'], capture_output=True, text=True, check=False)
65+
changed_files = [p for p in diff.stdout.splitlines() if p.strip()]
66+
67+
# Define core change (run all suites)
68+
def is_core_change(path: str) -> bool:
69+
if path.startswith('typescript-sdk/packages/'):
70+
return True
71+
if path in (
72+
'typescript-sdk/package.json',
73+
'typescript-sdk/pnpm-workspace.yaml',
74+
'typescript-sdk/tsconfig.json',
75+
'typescript-sdk/turbo.json',
76+
):
77+
return True
78+
if path.startswith('python-sdk/'):
79+
return True
80+
if path.startswith('typescript-sdk/apps/dojo/scripts/'):
81+
return True
82+
if path.startswith('typescript-sdk/apps/dojo/e2e/') and not path.startswith('typescript-sdk/apps/dojo/e2e/tests/'):
83+
return True
84+
if path == '.github/workflows/dojo-e2e.yml':
85+
return True
86+
return False
87+
88+
core_changed = any(is_core_change(p) for p in changed_files)
89+
90+
tests_map = {
91+
'agnoTests': ['agno'],
92+
'crewAITests': ['crew-ai'],
93+
'langgraphTests': ['langgraph'],
94+
'langgraphFastAPITests': ['langgraph-fastapi'],
95+
'llamaIndexTests': ['llama-index'],
96+
'mastraTests': ['mastra'],
97+
'mastraAgentLocalTests': ['mastra-agent-local'],
98+
'middlewareStarterTests': ['middleware-starter'],
99+
'pydanticAITests': ['pydantic-ai'],
100+
'serverStarterTests': ['server-starter'],
101+
'serverStarterAllFeaturesTests': ['server-starter-all'],
102+
'vercelAISdkTests': ['vercel-ai-sdk'],
103+
}
104+
105+
integrations_map = {
106+
'agno': ['agno'],
107+
'crewai': ['crew-ai'],
108+
'langgraph': ['langgraph', 'langgraph-fastapi'],
109+
'llamaindex': ['llama-index'],
110+
'mastra': ['mastra', 'mastra-agent-local'],
111+
'middleware-starter': ['middleware-starter'],
112+
'pydantic-ai': ['pydantic-ai'],
113+
'server-starter': ['server-starter'],
114+
'server-starter-all-features': ['server-starter-all'],
115+
'vercel-ai-sdk': ['vercel-ai-sdk'],
116+
}
117+
118+
suites = set()
149119
if core_changed:
150-
include = all_entries
120+
suites.update(e['suite'] for e in all_entries)
151121
else:
152-
if os.environ.get('AGNO') == 'true':
153-
include.append(entry_by_suite['agno'])
154-
if os.environ.get('CREW_AI') == 'true':
155-
include.append(entry_by_suite['crew-ai'])
156-
if os.environ.get('LANGGRAPH') == 'true':
157-
include.append(entry_by_suite['langgraph'])
158-
include.append(entry_by_suite['langgraph-fastapi'])
159-
if os.environ.get('LLAMA_INDEX') == 'true':
160-
include.append(entry_by_suite['llama-index'])
161-
if os.environ.get('MASTRA') == 'true':
162-
include.append(entry_by_suite['mastra'])
163-
include.append(entry_by_suite['mastra-agent-local'])
164-
if os.environ.get('MIDDLEWARE_STARTER') == 'true':
165-
include.append(entry_by_suite['middleware-starter'])
166-
if os.environ.get('PYDANTIC_AI') == 'true':
167-
include.append(entry_by_suite['pydantic-ai'])
168-
if os.environ.get('SERVER_STARTER') == 'true':
169-
include.append(entry_by_suite['server-starter'])
170-
if os.environ.get('SERVER_STARTER_ALL') == 'true':
171-
include.append(entry_by_suite['server-starter-all'])
172-
if os.environ.get('VERCEL_AI_SDK') == 'true':
173-
include.append(entry_by_suite['vercel-ai-sdk'])
174-
175-
# Include suites whose tests changed directly
176-
if os.environ.get('E2E_AGNO_TESTS') == 'true':
177-
include.append(entry_by_suite['agno'])
178-
if os.environ.get('E2E_CREW_AI_TESTS') == 'true':
179-
include.append(entry_by_suite['crew-ai'])
180-
if os.environ.get('E2E_LANGGRAPH_TESTS') == 'true':
181-
include.append(entry_by_suite['langgraph'])
182-
if os.environ.get('E2E_LANGGRAPH_FASTAPI_TESTS') == 'true':
183-
include.append(entry_by_suite['langgraph-fastapi'])
184-
if os.environ.get('E2E_LLAMA_INDEX_TESTS') == 'true':
185-
include.append(entry_by_suite['llama-index'])
186-
if os.environ.get('E2E_MASTRA_TESTS') == 'true':
187-
include.append(entry_by_suite['mastra'])
188-
if os.environ.get('E2E_MASTRA_AGENT_LOCAL_TESTS') == 'true':
189-
include.append(entry_by_suite['mastra-agent-local'])
190-
if os.environ.get('E2E_MIDDLEWARE_STARTER_TESTS') == 'true':
191-
include.append(entry_by_suite['middleware-starter'])
192-
if os.environ.get('E2E_PYDANTIC_AI_TESTS') == 'true':
193-
include.append(entry_by_suite['pydantic-ai'])
194-
if os.environ.get('E2E_SERVER_STARTER_TESTS') == 'true':
195-
include.append(entry_by_suite['server-starter'])
196-
if os.environ.get('E2E_SERVER_STARTER_ALL_TESTS') == 'true':
197-
include.append(entry_by_suite['server-starter-all'])
198-
if os.environ.get('E2E_VERCEL_AI_SDK_TESTS') == 'true':
199-
include.append(entry_by_suite['vercel-ai-sdk'])
122+
for path in changed_files:
123+
if path.startswith('typescript-sdk/apps/dojo/e2e/tests/'):
124+
parts = path.split('/')
125+
if len(parts) > 5:
126+
test_dir = parts[5]
127+
suites.update(tests_map.get(test_dir, []))
128+
if path.startswith('typescript-sdk/integrations/'):
129+
parts = path.split('/')
130+
if len(parts) > 2:
131+
integ = parts[2]
132+
suites.update(integrations_map.get(integ, []))
200133
201-
# de-duplicate by suite in case multiple reasons include the same suite
202-
include_unique = list({e['suite']: e for e in include}.values())
134+
include_unique = [entry_by_suite[s] for s in sorted(suites) if s in entry_by_suite]
203135
matrix = {"include": include_unique}
204136
with open(os.environ['GITHUB_OUTPUT'], 'a') as fh:
205137
fh.write(f"matrix={json.dumps(matrix)}\n")
206-
fh.write(f"should_run={'true' if include else 'false'}\n")
138+
fh.write(f"should_run={'true' if include_unique else 'false'}\n")
207139
PY
208140
e2e:
209141
needs: changes

0 commit comments

Comments
 (0)