ci: more granular test filtering and better testing structure

.github/workflows/check-generated-files.yml

@@ -1,14 +1,21 @@
-name: Check Generated Files
+name: static
 
 on:
   push:
     branches: [main]
+    paths:
+      - "apps/dojo/**"
+      - "integrations/**"
+      - ".github/workflows/check-generated-files.yml"
   pull_request:
     branches: [main]
+    paths:
+      - "apps/dojo/**"
+      - "integrations/**"
+      - ".github/workflows/check-generated-files.yml"
 
 jobs:
-  check-files-json:
-    name: Check files.json
+  check-dojo-generated-files:
     runs-on: ubuntu-latest
     steps:
       - name: Checkout code
@@ -17,7 +24,7 @@ jobs:
       - name: Set up Node.js
         uses: actions/setup-node@v4
         with:
-          node-version: '22'
+          node-version: "22"
 
       - name: Install pnpm
         uses: pnpm/action-setup@v4

.github/workflows/dojo-e2e.yml

@@ -3,52 +3,66 @@ name: e2e
 on:
   push:
     branches: [main]
+    paths:
+      - "integrations/**"
+      - "apps/**"
+      - "middlewares/**"
+      - "pnpm-lock.yaml"
+      - "pnpm-workspace.yaml"
+      - ".github/workflows/dojo-e2e.yml"
   pull_request:
     branches: [main]
+    paths:
+      - "integrations/**"
+      - "apps/**"
+      - "middlewares/**"
+      - "pnpm-lock.yaml"
+      - "pnpm-workspace.yaml"
+      - ".github/workflows/dojo-e2e.yml"
 
 jobs:
-  e2e:
-    name: ${{ matrix.suite }}
+  dojo:
+    name: dojo / ${{ matrix.suite }}
     runs-on: depot-ubuntu-24.04
     strategy:
       fail-fast: false
       matrix:
         include:
           - suite: a2a-middleware
             test_path: tests/a2aMiddlewareTests
-            services: ["dojo","a2a-middleware"]
+            services: ["dojo", "a2a-middleware"]
             wait_on: http://localhost:9999,tcp:localhost:8011,tcp:localhost:8012,tcp:localhost:8013,tcp:localhost:8014
           - suite: adk-middleware
             test_path: tests/adkMiddlewareTests
-            services: ["dojo","adk-middleware"]
+            services: ["dojo", "adk-middleware"]
             wait_on: http://localhost:9999,tcp:localhost:8010
           - suite: agno
             test_path: tests/agnoTests
-            services: ["dojo","agno"]
+            services: ["dojo", "agno"]
             wait_on: http://localhost:9999,tcp:localhost:8002
           - suite: crew-ai
             test_path: tests/crewAITests
-            services: ["dojo","crew-ai"]
+            services: ["dojo", "crew-ai"]
             wait_on: http://localhost:9999,tcp:localhost:8003
           - suite: langgraph-python
             test_path: tests/langgraphPythonTests
-            services: ["dojo","langgraph-platform-python"]
+            services: ["dojo", "langgraph-platform-python"]
             wait_on: http://localhost:9999,tcp:localhost:8005
           - suite: langgraph-typescript
             test_path: tests/langgraphTypescriptTests
-            services: ["dojo","langgraph-platform-typescript"]
+            services: ["dojo", "langgraph-platform-typescript"]
             wait_on: http://localhost:9999,tcp:localhost:8006
           - suite: langgraph-fastapi
             test_path: tests/langgraphFastAPITests
-            services: ["dojo","langgraph-fastapi"]
+            services: ["dojo", "langgraph-fastapi"]
             wait_on: http://localhost:9999,tcp:localhost:8004
           - suite: llama-index
             test_path: tests/llamaIndexTests
-            services: ["dojo","llama-index"]
+            services: ["dojo", "llama-index"]
             wait_on: http://localhost:9999,tcp:localhost:8007
           - suite: mastra
             test_path: tests/mastraTests
-            services: ["dojo","mastra"]
+            services: ["dojo", "mastra"]
             wait_on: http://localhost:9999,tcp:localhost:8008
           - suite: mastra-agent-local
             test_path: tests/mastraAgentLocalTests
@@ -60,125 +74,125 @@ jobs:
             wait_on: http://localhost:9999
           - suite: pydantic-ai
             test_path: tests/pydanticAITests
-            services: ["dojo","pydantic-ai"]
+            services: ["dojo", "pydantic-ai"]
             wait_on: http://localhost:9999,tcp:localhost:8009
           - suite: server-starter
             test_path: tests/serverStarterTests
-            services: ["dojo","server-starter"]
+            services: ["dojo", "server-starter"]
             wait_on: http://localhost:9999,tcp:localhost:8000
           - suite: server-starter-all
             test_path: tests/serverStarterAllFeaturesTests
-            services: ["dojo","server-starter-all"]
+            services: ["dojo", "server-starter-all"]
             wait_on: http://localhost:9999,tcp:localhost:8001
           # - suite: vercel-ai-sdk
           #   test_path: tests/vercelAISdkTests
           #   services: ["dojo"]
           #   wait_on: http://localhost:9999
 
     steps:
-    - name: Checkout code
-      uses: actions/checkout@v4
-
-    - name: Set up Node.js
-      uses: actions/setup-node@v4
-      with:
-        node-version: '22'
-
-    - name: Install pnpm
-      uses: pnpm/action-setup@v4
-      with:
-        version: 10.13.1
-
-    # Now that pnpm is available, cache its store to speed installs
-    - name: Resolve pnpm store path
-      id: pnpm-store
-      run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
-
-    - name: Cache pnpm store
-      uses: actions/cache@v4
-      with:
-        path: ${{ env.STORE_PATH }}
-        key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
-        restore-keys: |
-          ${{ runner.os }}-pnpm-store-
-
-    # Cache Python tool caches and virtualenvs; restore only to avoid long saves
-    - name: Cache Python dependencies (restore-only)
-      id: cache-python
-      uses: actions/cache/restore@v4
-      with:
-        path: |
-          ~/.cache/pip
-          ~/.cache/pypoetry
-          ~/.cache/uv
-          **/.venv
-        key: ${{ runner.os }}-pydeps-${{ hashFiles('**/poetry.lock', '**/pyproject.toml') }}
-        restore-keys: |
-          ${{ runner.os }}-pydeps-
-
-    - name: Install Poetry
-      uses: snok/install-poetry@v1
-      with:
-        version: latest
-        virtualenvs-create: true
-        virtualenvs-in-project: true
-
-    - name: Install uv
-      uses: astral-sh/setup-uv@v6
-
-    - name: Install dependencies
-      run: pnpm install --frozen-lockfile
-
-    - name: Prepare dojo for e2e
-      working-directory: apps/dojo
-      if: ${{ join(matrix.services, ',') != '' }}
-      run: node ./scripts/prep-dojo-everything.js --only ${{ join(matrix.services, ',') }}
-
-    - name: Install e2e dependencies
-      working-directory: apps/dojo/e2e
-      run: |
-        pnpm install
-
-    - name: write langgraph env files
-      working-directory: integrations/langgraph
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
-      if: ${{ contains(join(matrix.services, ','), 'langgraph-fastapi') || contains(join(matrix.services, ','), 'langgraph-platform-python') || contains(join(matrix.services, ','), 'langgraph-platform-typescript') }}
-      run: |
-        echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > python/examples/.env
-        echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> python/examples/.env
-        echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > typescript/examples/.env
-        echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> typescript/examples/.env
-
-    - name: Run dojo+agents
-      uses: JarvusInnovations/background-action@v1
-      env:
-        OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
-        LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
-        GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
-      if: ${{ join(matrix.services, ',') != '' && contains(join(matrix.services, ','), 'dojo') }}
-      with:
+      - name: Checkout code
+        uses: actions/checkout@v4
+
+      - name: Set up Node.js
+        uses: actions/setup-node@v4
+        with:
+          node-version: "22"
+
+      - name: Install pnpm
+        uses: pnpm/action-setup@v4
+        with:
+          version: 10.13.1
+
+      # Now that pnpm is available, cache its store to speed installs
+      - name: Resolve pnpm store path
+        id: pnpm-store
+        run: echo "STORE_PATH=$(pnpm store path --silent)" >> $GITHUB_ENV
+
+      - name: Cache pnpm store
+        uses: actions/cache@v4
+        with:
+          path: ${{ env.STORE_PATH }}
+          key: ${{ runner.os }}-pnpm-store-${{ hashFiles('**/pnpm-lock.yaml') }}
+          restore-keys: |
+            ${{ runner.os }}-pnpm-store-
+
+      # Cache Python tool caches and virtualenvs; restore only to avoid long saves
+      - name: Cache Python dependencies (restore-only)
+        id: cache-python
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cache/pip
+            ~/.cache/pypoetry
+            ~/.cache/uv
+            **/.venv
+          key: ${{ runner.os }}-pydeps-${{ hashFiles('**/poetry.lock', '**/pyproject.toml') }}
+          restore-keys: |
+            ${{ runner.os }}-pydeps-
+
+      - name: Install Poetry
+        uses: snok/install-poetry@v1
+        with:
+          version: latest
+          virtualenvs-create: true
+          virtualenvs-in-project: true
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v6
+
+      - name: Install dependencies
+        run: pnpm install --frozen-lockfile
+
+      - name: Prepare dojo for e2e
+        working-directory: apps/dojo
+        if: ${{ join(matrix.services, ',') != '' }}
+        run: node ./scripts/prep-dojo-everything.js --only ${{ join(matrix.services, ',') }}
+
+      - name: Install e2e dependencies
+        working-directory: apps/dojo/e2e
+        run: |
+          pnpm install
+
+      - name: write langgraph env files
+        working-directory: integrations/langgraph
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
+        if: ${{ contains(join(matrix.services, ','), 'langgraph-fastapi') || contains(join(matrix.services, ','), 'langgraph-platform-python') || contains(join(matrix.services, ','), 'langgraph-platform-typescript') }}
         run: |
-          node ../scripts/run-dojo-everything.js --only ${{ join(matrix.services, ',') }}
+          echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > python/examples/.env
+          echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> python/examples/.env
+          echo "OPENAI_API_KEY=${OPENAI_API_KEY}" > typescript/examples/.env
+          echo "LANGSMITH_API_KEY=${LANGSMITH_API_KEY}" >> typescript/examples/.env
+
+      - name: Run dojo+agents
+        uses: JarvusInnovations/background-action@v1
+        env:
+          OPENAI_API_KEY: ${{ secrets.OPENAI_API_KEY }}
+          LANGSMITH_API_KEY: ${{ secrets.LANGSMITH_API_KEY }}
+          GOOGLE_API_KEY: ${{ secrets.GOOGLE_API_KEY }}
+        if: ${{ join(matrix.services, ',') != '' && contains(join(matrix.services, ','), 'dojo') }}
+        with:
+          run: |
+            node ../scripts/run-dojo-everything.js --only ${{ join(matrix.services, ',') }}
+          working-directory: apps/dojo/e2e
+          wait-on: ${{ matrix.wait_on }}
+          wait-for: 300000
+
+      - name: Run tests – ${{ matrix.suite }}
         working-directory: apps/dojo/e2e
-        wait-on: ${{ matrix.wait_on }}
-        wait-for: 300000
-
-    - name: Run tests – ${{ matrix.suite }}
-      working-directory: apps/dojo/e2e
-      env:
-        BASE_URL: http://localhost:9999
-        PLAYWRIGHT_SUITE: ${{ matrix.suite }}
-      run: |
-        pnpm test -- ${{ matrix.test_path }}
-
-    - name: Upload traces – ${{ matrix.suite }}
-      if: always() # Uploads artifacts even if tests fail
-      uses: actions/upload-artifact@v4
-      with:
-        name: ${{ matrix.suite }}-playwright-traces
-        path: |
-          apps/dojo/e2e/test-results/${{ matrix.suite }}/**/*
-          apps/dojo/e2e/playwright-report/**/*
-        retention-days: 7
+        env:
+          BASE_URL: http://localhost:9999
+          PLAYWRIGHT_SUITE: ${{ matrix.suite }}
+        run: |
+          pnpm test -- ${{ matrix.test_path }}
+
+      - name: Upload traces – ${{ matrix.suite }}
+        if: always() # Uploads artifacts even if tests fail
+        uses: actions/upload-artifact@v4
+        with:
+          name: ${{ matrix.suite }}-playwright-traces
+          path: |
+            apps/dojo/e2e/test-results/${{ matrix.suite }}/**/*
+            apps/dojo/e2e/playwright-report/**/*
+          retention-days: 7