New era of e2e Tests Bittensor

.github/workflows/e2e-subtensor-tests.yaml

@@ -25,7 +25,7 @@ env:
 
 # job to run tests in parallel
 jobs:
-  # Job to find all test files
+
   find-tests:
     runs-on: ubuntu-latest
     if: ${{ github.event_name != 'pull_request' || github.event.pull_request.draft == false }}
@@ -42,68 +42,55 @@ jobs:
           echo "::set-output name=test-files::$test_files"
         shell: bash
 
+  pull-docker-image:
+    runs-on: ubuntu-latest
+    steps:
+      - name: Log in to GitHub Container Registry
+        run: echo "${{ secrets.GITHUB_TOKEN }}" | docker login ghcr.io -u $GITHUB_ACTOR --password-stdin
+
+      - name: Pull Docker Image
+        run: docker pull ghcr.io/opentensor/subtensor-localnet:latest
+
+      - name: Save Docker Image to Cache
+        run: docker save -o subtensor-localnet.tar ghcr.io/opentensor/subtensor-localnet:latest
+
+      - name: Upload Docker Image as Artifact
+        uses: actions/upload-artifact@v4
+        with:
+          name: subtensor-localnet
+          path: subtensor-localnet.tar
+
   # Job to run tests in parallel
   run:
-    needs: find-tests
-    runs-on: SubtensorCI
+    needs:
+      - find-tests
+      - pull-docker-image
+    runs-on: ubuntu-latest
     timeout-minutes: 45
     strategy:
       fail-fast: false  # Allow other matrix jobs to run even if this job fails
-      max-parallel: 8  # Set the maximum number of parallel jobs
+      max-parallel: 32  # Set the maximum number of parallel jobs (same as we have cores in SubtensorCI runner)
       matrix:
-        rust-branch:
-          - stable
-        rust-target:
-          - x86_64-unknown-linux-gnu
         os:
           - ubuntu-latest
         test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }}
-    env:
-      RELEASE_NAME: development
-      RUSTV: ${{ matrix.rust-branch }}
-      RUST_BACKTRACE: full
-      RUST_BIN_DIR: target/${{ matrix.rust-target }}
-      TARGET: ${{ matrix.rust-target }}
     steps:
-      - name: Check-out repository under $GITHUB_WORKSPACE
+      - name: Check-out repository
         uses: actions/checkout@v4
 
-      - name: Install dependencies
-        run: |
-          sudo apt-get update &&
-          sudo apt-get install -y clang curl libssl-dev llvm libudev-dev protobuf-compiler
-
-      - name: Install Rust ${{ matrix.rust-branch }}
-        uses: actions-rs/toolchain@v1.0.6
-        with:
-          toolchain: ${{ matrix.rust-branch }}
-          components: rustfmt
-          profile: minimal
-
-      - name: Add wasm32-unknown-unknown target
-        run: |
-          rustup target add wasm32-unknown-unknown --toolchain stable-x86_64-unknown-linux-gnu
-          rustup component add rust-src --toolchain stable-x86_64-unknown-linux-gnu
-
-      - name: Clone subtensor repo
-        run: git clone https://github.com/opentensor/subtensor.git
-
-      - name: Setup subtensor repo
-        working-directory: ${{ github.workspace }}/subtensor
-        run: git checkout devnet-ready
-
       - name: Install uv
         uses: astral-sh/setup-uv@v4
 
       - name: install dependencies
         run: uv sync --all-extras --dev
 
-      - name: Run tests
-        run: |
-          LOCALNET_SH_PATH="${{ github.workspace }}/subtensor/scripts/localnet.sh" uv run pytest ${{ matrix.test-file }} -s
+      - name: Download Cached Docker Image
+        uses: actions/download-artifact@v4
+        with:
+          name: subtensor-localnet
 
-      - name: Retry failed tests
-        if: failure()
-        run: |
-          sleep 10
-          LOCALNET_SH_PATH="${{ github.workspace }}/subtensor/scripts/localnet.sh" uv run pytest ${{ matrix.test-file }} -s
+      - name: Load Docker Image
+        run: docker load -i subtensor-localnet.tar
+
+      - name: Run tests
+        run: uv run pytest ${{ matrix.test-file }} -s

tests/e2e_tests/conftest.py

@@ -1,27 +1,82 @@
 import os
 import re
+import shutil
 import shlex
 import signal
 import subprocess
-import time
+import sys
 import threading
+import time
+from bittensor.utils.btlogging import logging
 
 import pytest
 from async_substrate_interface import SubstrateInterface
 
 from bittensor.core.async_subtensor import AsyncSubtensor
 from bittensor.core.subtensor import Subtensor
-from bittensor.utils.btlogging import logging
 from tests.e2e_tests.utils.e2e_test_utils import (
     Templates,
     setup_wallet,
 )
 
 
-# Fixture for setting up and tearing down a localnet.sh chain between tests
+def wait_for_node_start(process, timestamp=None):
+    """Waits for node to start in the docker."""
+    while True:
+        line = process.stdout.readline()
+        if not line:
+            break
+
+        timestamp = timestamp or int(time.time())
+        print(line.strip())
+        # 10 min as timeout
+        if int(time.time()) - timestamp > 20 * 30:
+            print("Subtensor not started in time")
+            raise TimeoutError
+
+        pattern = re.compile(r"Imported #1")
+        if pattern.search(line):
+            print("Node started!")
+            break
+
+    # Start a background reader after pattern is found
+    # To prevent the buffer filling up
+    def read_output():
+        while True:
+            if not process.stdout.readline():
+                break
+
+    reader_thread = threading.Thread(target=read_output, daemon=True)
+    reader_thread.start()
+
+
 @pytest.fixture(scope="function")
 def local_chain(request):
-    param = request.param if hasattr(request, "param") else None
+    """Determines whether to run the localnet.sh script in a subprocess or a Docker container."""
+    args = request.param if hasattr(request, "param") else None
+    params = "" if args is None else f"{args}"
+    if shutil.which("docker"):
+        yield from docker_runner(params)
+        return
+
+    if sys.platform.startswith("linux"):
+        docker_commend = (
+            "Install docker with command "
+            "[blue]sudo apt-get update && sudo apt-get install docker.io -y[/blue]"
+        )
+    elif sys.platform == "darwin":
+        docker_commend = "Install docker with command [blue]brew install docker[/blue]"
+    else:
+        docker_commend = "[blue]Unknown OS, install Docker manually: https://docs.docker.com/get-docker/[/blue]"
+
+    logging.warning("Docker not found in the operating system!")
+    logging.warning(docker_commend)
+    logging.warning("Tests are run in legacy mode.")
+    yield from legacy_runner(request)
+
+
+def legacy_runner(params):
+    """Runs the localnet.sh script in a subprocess and waits for it to start."""
     # Get the environment variable for the script path
     script_path = os.getenv("LOCALNET_SH_PATH")
 
@@ -31,41 +86,11 @@ def local_chain(request):
         pytest.skip("LOCALNET_SH_PATH environment variable is not set.")
 
     # Check if param is None, and handle it accordingly
-    args = "" if param is None else f"{param}"
+    args = "" if params is None else f"{params}"
 
     # Compile commands to send to process
     cmds = shlex.split(f"{script_path} {args}")
 
-    # Pattern match indicates node is compiled and ready
-    pattern = re.compile(r"Imported #1")
-    timestamp = int(time.time())
-
-    def wait_for_node_start(process, pattern):
-        while True:
-            line = process.stdout.readline()
-            if not line:
-                break
-
-            print(line.strip())
-            # 10 min as timeout
-            if int(time.time()) - timestamp > 20 * 60:
-                print("Subtensor not started in time")
-                raise TimeoutError
-            if pattern.search(line):
-                print("Node started!")
-                break
-
-        # Start a background reader after pattern is found
-        # To prevent the buffer filling up
-        def read_output():
-            while True:
-                line = process.stdout.readline()
-                if not line:
-                    break
-
-        reader_thread = threading.Thread(target=read_output, daemon=True)
-        reader_thread.start()
-
     with subprocess.Popen(
         cmds,
         start_new_session=True,
@@ -74,7 +99,7 @@ def read_output():
         text=True,
     ) as process:
         try:
-            wait_for_node_start(process, pattern)
+            wait_for_node_start(process)
         except TimeoutError:
             raise
         else:
@@ -91,6 +116,59 @@ def read_output():
                 process.wait()
 
 
+def docker_runner(params):
+    """Starts a Docker container before tests and gracefully terminates it after."""
+
+    container_name = f"test_local_chain_{str(time.time()).replace(".", "_")}"
+    image_name = "ghcr.io/opentensor/subtensor-localnet:latest"
+
+    # Command to start container
+    cmds = [
+        "docker",
+        "run",
+        "--rm",
+        "--name",
+        container_name,
+        "-p",
+        "9944:9944",
+        "-p",
+        "9945:9945",
+        image_name,
+        params,
+    ]
+
+    # Start container
+    with subprocess.Popen(
+        cmds,
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+        start_new_session=True,
+    ) as process:
+        try:
+            try:
+                wait_for_node_start(process, int(time.time()))
+            except TimeoutError:
+                raise
+
+            result = subprocess.run(
+                ["docker", "ps", "-q", "-f", f"name={container_name}"],
+                capture_output=True,
+                text=True,
+            )
+            if not result.stdout.strip():
+                raise RuntimeError("Docker container failed to start.")
+
+            yield SubstrateInterface(url="ws://127.0.0.1:9944")
+
+        finally:
+            try:
+                subprocess.run(["docker", "kill", container_name])
+                process.wait()
+            except subprocess.TimeoutExpired:
+                os.killpg(os.getpgid(process.pid), signal.SIGKILL)
+
+
 @pytest.fixture(scope="session")
 def templates():
     with Templates() as templates: