ci: further refining CI logic for change detection #378
Workflow file for this run
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: e2e | |
| on: | |
| push: | |
| branches: [main] | |
| pull_request: | |
| branches: [main] | |
| jobs: | |
| changes: | |
| name: Determine suites to run | |
| runs-on: ubuntu-latest | |
| outputs: | |
| matrix: ${{ steps.set-matrix.outputs.matrix }} | |
| should_run: ${{ steps.set-matrix.outputs.should_run }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| with: | |
| fetch-depth: 0 | |
| # Simplified: compute changed suites via git diff in the next step | |
| - name: Build dynamic matrix | |
| id: set-matrix | |
| env: | |
| EVENT_NAME: ${{ github.event_name }} | |
| PR_BASE_SHA: ${{ github.event.pull_request.base.sha }} | |
| BEFORE_SHA: ${{ github.event.before }} | |
| HEAD_SHA: ${{ github.sha }} | |
| run: | | |
| python3 - << 'PY' | |
| import os, json, subprocess | |
| all_entries = [ | |
| {"suite": "agno", "test_path": "tests/agnoTests", "services": ["dojo","agno"], "wait_on": "http://localhost:9999,tcp:localhost:8002"}, | |
| {"suite": "crew-ai", "test_path": "tests/crewAITests", "services": ["dojo","crew-ai"], "wait_on": "http://localhost:9999,tcp:localhost:8003"}, | |
| {"suite": "langgraph", "test_path": "tests/langgraphTests", "services": ["dojo","langgraph-platform-python","langgraph-platform-typescript"], "wait_on": "http://localhost:9999,tcp:localhost:8005,tcp:localhost:8006"}, | |
| {"suite": "langgraph-fastapi", "test_path": "tests/langgraphFastAPITests", "services": ["dojo","langgraph-fastapi"], "wait_on": "http://localhost:9999,tcp:localhost:8004"}, | |
| {"suite": "llama-index", "test_path": "tests/llamaIndexTests", "services": ["dojo","llama-index"], "wait_on": "http://localhost:9999,tcp:localhost:8007"}, | |
| {"suite": "mastra", "test_path": "tests/mastraTests", "services": ["dojo","mastra"], "wait_on": "http://localhost:9999,tcp:localhost:8008"}, | |
| {"suite": "mastra-agent-local", "test_path": "tests/mastraAgentLocalTests", "services": ["dojo"], "wait_on": "http://localhost:9999"}, | |
| {"suite": "middleware-starter", "test_path": "tests/middlewareStarterTests", "services": ["dojo"], "wait_on": "http://localhost:9999"}, | |
| {"suite": "pydantic-ai", "test_path": "tests/pydanticAITests", "services": ["dojo","pydantic-ai"], "wait_on": "http://localhost:9999,tcp:localhost:8009"}, | |
| {"suite": "server-starter", "test_path": "tests/serverStarterTests", "services": ["dojo","server-starter"], "wait_on": "http://localhost:9999,tcp:localhost:8000"}, | |
| {"suite": "server-starter-all", "test_path": "tests/serverStarterAllFeaturesTests", "services": ["dojo","server-starter-all"], "wait_on": "http://localhost:9999,tcp:localhost:8001"}, | |
| {"suite": "vercel-ai-sdk", "test_path": "tests/vercelAISdkTests", "services": ["dojo"], "wait_on": "http://localhost:9999"}, | |
| ] | |
| entry_by_suite = {e["suite"]: e for e in all_entries} | |
| # Determine changed files between base and head | |
| event_name = os.getenv('EVENT_NAME') | |
| base = os.getenv('PR_BASE_SHA') if event_name == 'pull_request' else os.getenv('BEFORE_SHA') | |
| head = os.getenv('HEAD_SHA') | |
| if not base: | |
| try: | |
| base = subprocess.check_output(['git', 'rev-parse', 'HEAD~1'], text=True).strip() | |
| except Exception: | |
| base = head | |
| subprocess.run(['git', 'fetch', '--no-tags', '--prune', '--depth=0', 'origin'], check=False) | |
| diff = subprocess.run(['git', 'diff', '--name-only', f'{base}...{head}'], capture_output=True, text=True, check=False) | |
| changed_files = [p for p in diff.stdout.splitlines() if p.strip()] | |
| # Define core change (run all suites) | |
| def is_core_change(path: str) -> bool: | |
| if path.startswith('typescript-sdk/packages/'): | |
| return True | |
| if path in ( | |
| 'typescript-sdk/package.json', | |
| 'typescript-sdk/pnpm-workspace.yaml', | |
| 'typescript-sdk/tsconfig.json', | |
| 'typescript-sdk/turbo.json', | |
| ): | |
| return True | |
| if path.startswith('python-sdk/'): | |
| return True | |
| if path.startswith('typescript-sdk/apps/dojo/scripts/'): | |
| return True | |
| if path.startswith('typescript-sdk/apps/dojo/e2e/') and not path.startswith('typescript-sdk/apps/dojo/e2e/tests/'): | |
| return True | |
| if path == '.github/workflows/dojo-e2e.yml': | |
| return True | |
| return False | |
| core_changed = any(is_core_change(p) for p in changed_files) | |
| tests_map = { | |
| 'agnoTests': ['agno'], | |
| 'crewAITests': ['crew-ai'], | |
| 'langgraphTests': ['langgraph'], | |
| 'langgraphFastAPITests': ['langgraph-fastapi'], | |
| 'llamaIndexTests': ['llama-index'], | |
| 'mastraTests': ['mastra'], | |
| 'mastraAgentLocalTests': ['mastra-agent-local'], | |
| 'middlewareStarterTests': ['middleware-starter'], | |
| 'pydanticAITests': ['pydantic-ai'], | |
| 'serverStarterTests': ['server-starter'], | |
| 'serverStarterAllFeaturesTests': ['server-starter-all'], | |
| 'vercelAISdkTests': ['vercel-ai-sdk'], | |
| } | |
| integrations_map = { | |
| 'agno': ['agno'], | |
| 'crewai': ['crew-ai'], | |
| 'langgraph': ['langgraph', 'langgraph-fastapi'], | |
| 'llamaindex': ['llama-index'], | |
| 'mastra': ['mastra', 'mastra-agent-local'], | |
| 'middleware-starter': ['middleware-starter'], | |
| 'pydantic-ai': ['pydantic-ai'], | |
| 'server-starter': ['server-starter'], | |
| 'server-starter-all-features': ['server-starter-all'], | |
| 'vercel-ai-sdk': ['vercel-ai-sdk'], | |
| } | |
| suites = set() | |
| if core_changed: | |
| suites.update(e['suite'] for e in all_entries) | |
| else: | |
| for path in changed_files: | |
| if path.startswith('typescript-sdk/apps/dojo/e2e/tests/'): | |
| parts = path.split('/') | |
| if len(parts) > 5: | |
| test_dir = parts[5] | |
| suites.update(tests_map.get(test_dir, [])) | |
| if path.startswith('typescript-sdk/integrations/'): | |
| parts = path.split('/') | |
| if len(parts) > 2: | |
| integ = parts[2] | |
| suites.update(integrations_map.get(integ, [])) | |
| include_unique = [entry_by_suite[s] for s in sorted(suites) if s in entry_by_suite] | |
| matrix = {"include": include_unique} | |
| with open(os.environ['GITHUB_OUTPUT'], 'a') as fh: | |
| fh.write(f"matrix={json.dumps(matrix)}\n") | |
| fh.write(f"should_run={'true' if include_unique else 'false'}\n") | |
| PY | |
| e2e: | |
| needs: changes | |
| if: ${{ needs.changes.outputs.should_run == 'true' }} | |
| name: ${{ matrix.suite }} | |
| runs-on: depot-ubuntu-24.04 | |
| strategy: | |
| fail-fast: false | |
| matrix: ${{ fromJSON(needs.changes.outputs.matrix) }} | |
| steps: | |
| - name: Checkout code | |
| uses: actions/checkout@v4 | |
| - name: Set up Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| - name: Install pnpm | |
| uses: pnpm/action-setup@v4 | |
| with: | |
| version: 10.13.1 | |
| # Now that pnpm is available, cache its store to speed installs | |
| - name: Resolve pnpm store path | |
| id: pnpm-store | |
| run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV | |
| - name: Cache pnpm store | |
| uses: actions/cache@v4 | |
| with: | |
| path: ${{ env.STORE_PATH }} | |
| key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pnpm-store- | |
| # Cache Python tool caches and virtualenvs; restore only to avoid long saves | |
| - name: Cache Python dependencies (restore-only) | |
| id: cache-python | |
| uses: actions/cache/restore@v4 | |
| with: | |
| path: | | |
| ~/.cache/pip | |
| ~/.cache/pypoetry | |
| ~/.cache/uv | |
| **/.venv | |
| key: ${{ runner.os }}-pydeps-${{ hashFiles('**/poetry.lock', '**/pyproject.toml') }} | |
| restore-keys: | | |
| ${{ runner.os }}-pydeps- | |
| - name: Install Poetry | |
| uses: snok/install-poetry@v1 | |
| with: | |
| version: latest | |
| virtualenvs-create: true | |
| virtualenvs-in-project: true | |
| - name: Install uv | |
| uses: astral-sh/setup-uv@v6 | |
| - name: Install dependencies | |
| working-directory: typescript-sdk | |
| run: pnpm install --frozen-lockfile | |
| - name: Prepare dojo for e2e | |
| working-directory: typescript-sdk/apps/dojo | |
| if: ${{ join(matrix.services, ',') != '' }} | |
| run: node ./scripts/prep-dojo-everything.js --only ${{ join(matrix.services, ',') }} | |
| - name: Install e2e dependencies | |
| working-directory: typescript-sdk/apps/dojo/e2e | |
| run: | | |
| pnpm install | |
| - name: write langgraph env files | |
| working-directory: typescript-sdk/integrations/langgraph | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | |
| if: ${{ contains(join(matrix.services, ','), 'langgraph-fastapi') || contains(join(matrix.services, ','), 'langgraph-platform-python') || contains(join(matrix.services, ','), 'langgraph-platform-typescript') }} | |
| run: | | |
| echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > examples/python/.env | |
| echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> examples/python/.env | |
| echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > examples/typescript/.env | |
| echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> examples/typescript/.env | |
| echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > python/ag_ui_langgraph/.env | |
| echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> python/ag_ui_langgraph/.env | |
| - name: Run dojo+agents | |
| uses: JarvusInnovations/background-action@v1 | |
| env: | |
| OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }} | |
| LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }} | |
| if: ${{ join(matrix.services, ',') != '' && contains(join(matrix.services, ','), 'dojo') }} | |
| with: | |
| run: | | |
| node ../scripts/run-dojo-everything.js --only ${{ join(matrix.services, ',') }} | |
| working-directory: typescript-sdk/apps/dojo/e2e | |
| wait-on: ${{ matrix.wait_on }} | |
| wait-for: 300000 | |
| - name: Run tests – ${{ matrix.suite }} | |
| working-directory: typescript-sdk/apps/dojo/e2e | |
| env: | |
| BASE_URL: http://localhost:9999 | |
| PLAYWRIGHT_SUITE: ${{ matrix.suite }} | |
| run: | | |
| pnpm test -- ${{ matrix.test_path }} | |
| - name: Upload traces – ${{ matrix.suite }} | |
| if: always() # Uploads artifacts even if tests fail | |
| uses: actions/upload-artifact@v4 | |
| with: | |
| name: ${{ matrix.suite }}-playwright-traces | |
| path: | | |
| typescript-sdk/apps/dojo/e2e/test-results/${{ matrix.suite }}/**/* | |
| typescript-sdk/apps/dojo/e2e/playwright-report/**/* | |
| retention-days: 7 |