Merge pull request #2288 from opentensor/feat/roman/improve-bt-e2e-test-workflow

Reduce Subtensor compilation time and run E2E tests a few times faster

Author: sam0x17

…kflows/check-bittensor-e2e-tests.yml.yml …/workflows/check-bittensor-e2e-tests.yml.github/workflows/check-bittensor-e2e-tests.yml.yml renamed to .github/workflows/check-bittensor-e2e-tests.yml .github/workflows/check-bittensor-e2e-tests.yml.yml renamed to .github/workflows/check-bittensor-e2e-tests.yml

@@ -28,7 +28,7 @@ jobs:
   check-label:
     runs-on: ubuntu-latest
     outputs:
-      skip-bittensor-e2e-tests: ${{ steps.get-labels.outputs.skip-bittensor-e2e-tests }}
+      skip-bittensor-e2e-tests: ${{ steps.get-labels.outputs.skip-bittensor-e2e-tests || steps.set-default.outputs.skip-bittensor-e2e-tests }}
     steps:
       - name: Install dependencies
         run: |
@@ -38,11 +38,12 @@ jobs:
       - name: Check out repository
         uses: actions/checkout@v4
         with:
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
 
       - name: Get labels from PR
         id: get-labels
+        if: github.event_name == 'pull_request'
         run: |
           LABELS=$(gh pr -R ${{ github.repository }} view ${{ github.event.pull_request.number }} --json labels --jq '.labels[].name')
           echo "Current labels: $LABELS"
@@ -54,6 +55,12 @@ jobs:
         env:
           GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
 
+      - name: Set default skip value for workflow_dispatch
+        id: set-default
+        if: github.event_name == 'workflow_dispatch'
+        run: |
+          echo "skip-bittensor-e2e-tests=false" >> $GITHUB_OUTPUT
+
   find-btcli-e2e-tests:
     needs: check-label
     if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
@@ -74,11 +81,44 @@ jobs:
         working-directory: ${{ github.workspace }}/btcli
         run: git checkout staging
 
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: "false"
+
+      - name: Create Python virtual environment
+        working-directory: ${{ github.workspace }}/btcli
+        run: uv venv --seed
+
+      - name: Install dependencies
+        working-directory: ${{ github.workspace }}/btcli
+        run: |
+          source .venv/bin/activate
+          uv run --active pip install --upgrade pip
+          uv run --active pip install '.[dev]'
+          uv run --active pip install pytest
+
       - name: Find e2e test files
         id: get-btcli-tests
+        working-directory: ${{ github.workspace }}/btcli
         run: |
-          test_files=$(find ${{ github.workspace }}/btcli/tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))')
-          echo "test-files=$test_files" >> $GITHUB_OUTPUT
+          set -euo pipefail
+          test_matrix=$(
+            uv run pytest -q --collect-only tests/e2e_tests \
+            | sed -n '/^tests\//p' \
+            | jq -R -s -c '
+                split("\n")
+                | map(select(. != ""))
+                | map({nodeid: ., label: (sub("^tests/e2e_tests/"; ""))})
+              '
+          )
+          echo "Found tests: $test_matrix"
+          echo "test-files=$test_matrix" >> "$GITHUB_OUTPUT"
         shell: bash
 
   find-sdk-e2e-tests:
@@ -101,34 +141,148 @@ jobs:
         working-directory: ${{ github.workspace }}/bittensor
         run: git checkout staging
 
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: '3.10'
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v5
+        with:
+          enable-cache: "false"
+
+      - name: Create Python virtual environment
+        working-directory: ${{ github.workspace }}/bittensor
+        run: uv venv --seed
+
+      - name: Install dependencies
+        working-directory: ${{ github.workspace }}/bittensor
+        run: |
+          source .venv/bin/activate
+          uv run --active pip install --upgrade pip
+          uv run --active pip install '.[dev]'
+          uv run --active pip install pytest
+
       - name: Find e2e test files
         id: get-sdk-tests
+        working-directory: ${{ github.workspace }}/bittensor
         run: |
-          test_files=$(find ${{ github.workspace }}/bittensor/tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))')
-          echo "test-files=$test_files" >> $GITHUB_OUTPUT
+          set -euo pipefail
+          test_matrix=$(
+            uv run pytest -q --collect-only tests/e2e_tests \
+            | sed -n '/^e2e_tests\//p' \
+            | sed 's|^|tests/|' \
+            | jq -R -s -c '
+                split("\n")
+                | map(select(. != ""))
+                | map({nodeid: ., label: (sub("^tests/e2e_tests/"; ""))})
+              '
+          )
+          echo "Found tests: $test_matrix"
+          echo "test-files=$test_matrix" >> "$GITHUB_OUTPUT"
         shell: bash
 
-  build-image-with-current-branch:
+  # build artifacts for fast-runtime and non-fast-runtime
+  artifacts:
+    name: Node • ${{ matrix.runtime }} • ${{ matrix.platform.arch }}
     needs: check-label
     if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
-    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        platform:
+          - runner: [self-hosted, type-ccx33]
+            triple: x86_64-unknown-linux-gnu
+            arch: amd64
+        runtime: ["fast-runtime", "non-fast-runtime"]
+
+    runs-on: ${{ matrix.platform.runner }}
+
     steps:
       - name: Checkout code
         uses: actions/checkout@v4
         with:
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
+
+      - name: Install Rust + dependencies
+        run: |
+          chmod +x ./scripts/install_build_env.sh
+          ./scripts/install_build_env.sh
 
-      - name: Patch non-fast-runtime node
+      - name: Add Rust target triple
+        run: |
+          source "$HOME/.cargo/env"
+          rustup target add ${{ matrix.platform.triple }}
+
+      - name: Patch limits for local run
         run: |
           chmod +x ./scripts/localnet_patch.sh
           ./scripts/localnet_patch.sh
 
-      - name: Set up QEMU
-        uses: docker/setup-qemu-action@v3
+      - name: Build binaries
+        run: |
+          export PATH="$HOME/.cargo/bin:$PATH"
+          export CARGO_BUILD_TARGET="${{ matrix.platform.triple }}"
+
+          if [ "${{ matrix.runtime }}" = "fast-runtime" ]; then
+            ./scripts/localnet.sh --build-only
+          else
+            ./scripts/localnet.sh False --build-only
+          fi
+
+      # use `ci_target` name bc .dockerignore excludes `target`
+      - name: Prepare artifacts for upload
+        run: |
+          RUNTIME="${{ matrix.runtime }}"
+          TRIPLE="${{ matrix.platform.triple }}"
+
+          # Verify binaries exist before copying
+          BINARY_PATH="target/${RUNTIME}/${TRIPLE}/release/node-subtensor"
+          WASM_PATH="target/${RUNTIME}/${TRIPLE}/release/wbuild/node-subtensor-runtime/node_subtensor_runtime.compact.compressed.wasm"
 
-      - name: Set up Docker Buildx
-        uses: docker/setup-buildx-action@v3
+          if [[ ! -f "$BINARY_PATH" ]]; then
+            echo "❌ Error: Binary not found at $BINARY_PATH"
+            exit 1
+          fi
+
+          if [[ ! -f "$WASM_PATH" ]]; then
+            echo "❌ Error: WASM file not found at $WASM_PATH"
+            exit 1
+          fi
+
+          mkdir -p build/ci_target/${RUNTIME}/${TRIPLE}/release/
+          cp -v "$BINARY_PATH" \
+            build/ci_target/${RUNTIME}/${TRIPLE}/release/
+
+          mkdir -p build/ci_target/${RUNTIME}/${TRIPLE}/release/wbuild/node-subtensor-runtime/
+          cp -v "$WASM_PATH" \
+            build/ci_target/${RUNTIME}/${TRIPLE}/release/wbuild/node-subtensor-runtime/
+
+      - name: Upload artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: binaries-${{ matrix.platform.triple }}-${{ matrix.runtime }}
+          path: build/
+          if-no-files-found: error
+
+  # Collect all artifacts and build a Docker image
+  build-image-with-current-branch:
+    needs: [check-label, artifacts]
+    if: needs.check-label.outputs.skip-bittensor-e2e-tests == 'false'
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
+
+      - name: Download all binary artifacts
+        uses: actions/download-artifact@v5
+        with:
+          pattern: binaries-*
+          path: build/
+          merge-multiple: true
 
       - name: Move Docker data-root to /mnt/data
         run: |
@@ -141,7 +295,7 @@ jobs:
           docker info | grep "Docker Root Dir"
 
       - name: Build Docker Image
-        run: docker build -f Dockerfile-localnet -t localnet .
+        run: docker build -f Dockerfile-localnet --build-arg BUILT_IN_CI="Boom shakalaka" -t localnet .
 
       - name: Save Docker Image as Tar
         run: docker save -o /mnt/data/subtensor-localnet.tar localnet
@@ -162,31 +316,18 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      max-parallel: 16
+      max-parallel: 32
       matrix:
-        rust-branch:
-          - stable
-        rust-target:
-          - x86_64-unknown-linux-gnu
-        os:
-          - ubuntu-latest
-        test-file: ${{ fromJson(needs.find-btcli-e2e-tests.outputs.test-files) }}
-
-    env:
-      RELEASE_NAME: development
-      RUSTV: ${{ matrix.rust-branch }}
-      RUST_BACKTRACE: full
-      RUST_BIN_DIR: target/${{ matrix.rust-target }}
-      TARGET: ${{ matrix.rust-target }}
+        include: ${{ fromJson(needs.find-btcli-e2e-tests.outputs.test-files) }}
 
     timeout-minutes: 60
-    name: "cli: ${{ matrix.test-file }}"
+    name: "cli: ${{ matrix.label }}"
     steps:
       - name: Check-out repository
         uses: actions/checkout@v4
         with:
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
 
       - name: Install uv
         uses: astral-sh/setup-uv@v5
@@ -230,7 +371,7 @@ jobs:
           set +e
           for i in 1 2; do
             echo "🔁 Attempt $i: Running tests"
-            uv run pytest ${{ matrix.test-file }} -s
+            uv run pytest "${{ matrix.nodeid }}" -s
             status=$?
             if [ $status -eq 0 ]; then
               echo "✅ Tests passed on attempt $i"
@@ -256,31 +397,18 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
-      max-parallel: 16
+      max-parallel: 64
       matrix:
-        rust-branch:
-          - stable
-        rust-target:
-          - x86_64-unknown-linux-gnu
-        os:
-          - ubuntu-latest
-        test-file: ${{ fromJson(needs.find-sdk-e2e-tests.outputs.test-files) }}
-
-    env:
-      RELEASE_NAME: development
-      RUSTV: ${{ matrix.rust-branch }}
-      RUST_BACKTRACE: full
-      RUST_BIN_DIR: target/${{ matrix.rust-target }}
-      TARGET: ${{ matrix.rust-target }}
+        include: ${{ fromJson(needs.find-sdk-e2e-tests.outputs.test-files) }}
 
     timeout-minutes: 60
-    name: "sdk: ${{ matrix.test-file }}"
+    name: "sdk: ${{ matrix.label }}"
     steps:
       - name: Check-out repository
         uses: actions/checkout@v4
         with:
-          repository: ${{ github.event.pull_request.head.repo.full_name }}
-          ref: ${{ github.event.pull_request.head.ref }}
+          repository: ${{ github.event.pull_request.head.repo.full_name || github.repository }}
+          ref: ${{ github.event.pull_request.head.ref || github.ref_name }}
 
       - name: Install uv
         uses: astral-sh/setup-uv@v5
@@ -324,7 +452,7 @@ jobs:
           set +e
           for i in 1 2; do
             echo "🔁 Attempt $i: Running tests"
-            uv run pytest ${{ matrix.test-file }} -s
+            uv run pytest "${{ matrix.nodeid }}" -s
             status=$?
             if [ $status -eq 0 ]; then
               echo "✅ Tests passed on attempt $i"