Merge pull request #2860 from opentensor/feat/roman/add-non-fast-block-tests-each-saturday

Add `non-fast-blocks` e2e tests each Saturday

Author: basfroman

.github/workflows/e2e-subtensor-tests.yaml

@@ -43,7 +43,7 @@ jobs:
         run: |
           test_files=$(find tests/e2e_tests -name "test*.py" | jq -R -s -c 'split("\n") | map(select(. != ""))')
           # keep it here for future debug
-          # test_files=$(find tests/e2e_tests -type f -name "test*.py" | grep -E 'test_(incentive|commit_weights|set_weights)\.py$' | jq -R -s -c 'split("\n") | map(select(. != ""))')
+          # test_files=$(find tests/e2e_tests -type f -name "test*.py" | grep -E 'test_(hotkeys|staking)\.py$' | jq -R -s -c 'split("\n") | map(select(. != ""))')
           echo "test-files=$test_files" >> "$GITHUB_OUTPUT"
         shell: bash
 
@@ -66,8 +66,8 @@ jobs:
           path: subtensor-localnet.tar
 
   # Job to run tests in parallel
-  run-e2e-test:
-    name: ${{ matrix.test-file }} / Python ${{ matrix.python-version }}
+  run-fast-blocks-e2e-test:
+    name: "FB: ${{ matrix.test-file }} / Python ${{ matrix.python-version }}"
     needs:
       - find-tests
       - pull-docker-image
@@ -127,3 +127,76 @@ jobs:
               sleep 5
             fi
           done
+
+
+  cron-run-non-fast-blocks-e2e-test:
+    if: github.event_name == 'schedule'
+    name: "NFB: ${{ matrix.test-file }} / Python ${{ matrix.python-version }}"
+    needs:
+      - find-tests
+      - pull-docker-image
+    runs-on: ubuntu-latest
+    timeout-minutes: 1440
+
+    strategy:
+      fail-fast: false  # Allow other matrix jobs to run even if this job fails
+      max-parallel: 32  # Set the maximum number of parallel jobs (same as we have cores in ubuntu-latest runner)
+      matrix:
+        os:
+          - ubuntu-latest
+        test-file: ${{ fromJson(needs.find-tests.outputs.test-files) }}
+        python-version: ["3.9", "3.10", "3.11", "3.12", "3.13"]
+
+    steps:
+      - name: Check if today is Saturday
+        run: |
+          day=$(date -u +%u)
+          echo "Today is weekday $day"
+          if [ "$day" -ne 6 ]; then
+            echo "⏭️ Skipping: not Saturday"
+            exit 78
+          fi
+      - name: Check-out repository
+        uses: actions/checkout@v4
+
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install uv
+        uses: astral-sh/setup-uv@v4
+
+      - name: install dependencies
+        run: uv sync --extra dev --dev
+
+      - name: Download Cached Docker Image
+        uses: actions/download-artifact@v4
+        with:
+          name: subtensor-localnet
+
+      - name: Load Docker Image
+        run: docker load -i subtensor-localnet.tar
+
+      - name: Run patched E2E tests
+        env:
+          FAST_BLOCKS: "0"
+        run: |
+          set +e
+          for i in 1 2 3; do
+            echo "🔁 Attempt $i: Running tests"
+            uv run pytest ${{ matrix.test-file }} -s
+            status=$?
+            if [ $status -eq 0 ]; then
+              echo "✅ Tests passed on attempt $i"
+              break
+            else
+              echo "❌ Tests failed on attempt $i"
+              if [ $i -eq 3 ]; then
+                echo "Tests failed after 3 attempts"
+                exit 1
+              fi
+              echo "Retrying..."
+              sleep 5
+            fi
+          done

bittensor/core/async_subtensor.py

@@ -2488,6 +2488,12 @@ async def immunity_period(
         )
         return None if call is None else int(call)
 
+    async def is_fast_blocks(self):
+        """Returns True if the node is running with fast blocks. False if not."""
+        return (
+            await self.query_constant("SubtensorModule", "DurationOfStartCall")
+        ).value == 10
+
     async def is_hotkey_delegate(
         self,
         hotkey_ss58: str,

bittensor/core/subtensor.py

@@ -1971,6 +1971,10 @@ def immunity_period(
         )
         return None if call is None else int(call)
 
+    def is_fast_blocks(self):
+        """Returns True if the node is running with fast blocks. False if not."""
+        return self.query_constant("SubtensorModule", "DurationOfStartCall").value == 10
+
     def is_hotkey_delegate(self, hotkey_ss58: str, block: Optional[int] = None) -> bool:
         """
         Determines whether a given hotkey (public key) is a delegate on the Bittensor network. This function checks if

bittensor/core/subtensor_api/chain.py

@@ -14,6 +14,7 @@ def __init__(self, subtensor: Union["_Subtensor", "_AsyncSubtensor"]):
         self.get_minimum_required_stake = subtensor.get_minimum_required_stake
         self.get_vote_data = subtensor.get_vote_data
         self.get_timestamp = subtensor.get_timestamp
+        self.is_fast_blocks = subtensor.is_fast_blocks
         self.last_drand_round = subtensor.last_drand_round
         self.state_call = subtensor.state_call
         self.tx_rate_limit = subtensor.tx_rate_limit

bittensor/core/subtensor_api/utils.py

@@ -101,6 +101,7 @@ def add_legacy_methods(subtensor: "SubtensorApi"):
     subtensor.get_unstake_fee = subtensor._subtensor.get_unstake_fee
     subtensor.get_vote_data = subtensor._subtensor.get_vote_data
     subtensor.immunity_period = subtensor._subtensor.immunity_period
+    subtensor.is_fast_blocks = subtensor._subtensor.is_fast_blocks
     subtensor.is_hotkey_delegate = subtensor._subtensor.is_hotkey_delegate
     subtensor.is_hotkey_registered = subtensor._subtensor.is_hotkey_registered
     subtensor.is_hotkey_registered_any = subtensor._subtensor.is_hotkey_registered_any

tests/e2e_tests/conftest.py

@@ -206,10 +206,13 @@ def stop_existing_test_containers():
         "9944:9944",
         "-p",
         "9945:9945",
-        LOCALNET_IMAGE_NAME,
-        params,
+        str(LOCALNET_IMAGE_NAME),
     ]
 
+    cmds += params.split() if params else []
+
+    print("Entire run command: ", cmds)
+
     try_start_docker()
 
     stop_existing_test_containers()
@@ -275,7 +278,19 @@ def bob_wallet():
     return wallet
 
 
+@pytest.fixture
+def charlie_wallet():
+    keypair, wallet = setup_wallet("//Charlie")
+    return wallet
+
+
 @pytest.fixture
 def dave_wallet():
     keypair, wallet = setup_wallet("//Dave")
     return wallet
+
+
+@pytest.fixture
+def eve_wallet():
+    keypair, wallet = setup_wallet("//Eve")
+    return wallet

tests/e2e_tests/test_commit_reveal_v3.py

@@ -30,17 +30,22 @@ async def test_commit_and_reveal_weights_cr3(local_chain, subtensor, alice_walle
     Raises:
         AssertionError: If any of the checks or verifications fail
     """
-    BLOCK_TIME = 0.25  # 12 for non-fast-block, 0.25 for fast block
-    netuid = 2
+    BLOCK_TIME = (
+        0.25 if subtensor.is_fast_blocks() else 12.0
+    )  # 12 for non-fast-block, 0.25 for fast block
+    netuid = subtensor.get_total_subnets()  # 2
+
     logging.console.info("Testing test_commit_and_reveal_weights")
 
     # Register root as Alice
     assert subtensor.register_subnet(alice_wallet), "Unable to register the subnet"
 
     # Verify subnet 2 created successfully
-    assert subtensor.subnet_exists(netuid), "Subnet wasn't created successfully"
+    assert subtensor.subnet_exists(netuid), (
+        f"Subnet {netuid} wasn't created successfully"
+    )
 
-    logging.console.info("Subnet 2 is registered")
+    logging.console.success(f"Subnet {netuid} is registered")
 
     # Enable commit_reveal on the subnet
     assert sudo_set_hyperparameter_bool(
@@ -74,7 +79,7 @@ async def test_commit_and_reveal_weights_cr3(local_chain, subtensor, alice_walle
     logging.console.info("sudo_set_weights_set_rate_limit executed: set to 0")
 
     # Change the tempo of the subnet
-    tempo_set = 50
+    tempo_set = 50 if subtensor.is_fast_blocks() else 10
     assert (
         sudo_set_admin_utils(
             local_chain,
@@ -103,7 +108,7 @@ async def test_commit_and_reveal_weights_cr3(local_chain, subtensor, alice_walle
     )
 
     # Wait for 2 tempos to pass as CR3 only reveals weights after 2 tempos + 1
-    subtensor.wait_for_block((tempo_set * 2) + 1)
+    subtensor.wait_for_block(tempo_set * 2 + 1)
 
     # Lower than this might mean weights will get revealed before we can check them
     if upcoming_tempo - current_block < 3:
@@ -117,7 +122,7 @@ async def test_commit_and_reveal_weights_cr3(local_chain, subtensor, alice_walle
     latest_drand_round = subtensor.last_drand_round()
     upcoming_tempo = next_tempo(current_block, tempo)
     logging.console.info(
-        f"Post first wait_interval (to ensure window isnt too low): {current_block}, next tempo: {upcoming_tempo}, drand: {latest_drand_round}"
+        f"Post first wait_interval (to ensure window isn't too low): {current_block}, next tempo: {upcoming_tempo}, drand: {latest_drand_round}"
     )
 
     # Commit weights
@@ -171,6 +176,7 @@ async def test_commit_and_reveal_weights_cr3(local_chain, subtensor, alice_walle
         subtensor,
         netuid=netuid,
         reporting_interval=1,
+        sleep=BLOCK_TIME,
     )
 
     # Fetch the latest drand pulse

tests/e2e_tests/test_commit_weights.py

@@ -27,7 +27,7 @@ async def test_commit_and_reveal_weights_legacy(local_chain, subtensor, alice_wa
         AssertionError: If any of the checks or verifications fail
     """
     netuid = subtensor.get_total_subnets()  # 2
-
+    set_tempo = 100 if subtensor.is_fast_blocks() else 10
     print("Testing test_commit_and_reveal_weights")
 
     # Register root as Alice
@@ -78,7 +78,7 @@ async def test_commit_and_reveal_weights_legacy(local_chain, subtensor, alice_wa
         call_function="sudo_set_tempo",
         call_params={
             "netuid": netuid,
-            "tempo": 100,
+            "tempo": set_tempo,
         },
     )
 
@@ -165,11 +165,11 @@ async def test_commit_weights_uses_next_nonce(local_chain, subtensor, alice_wall
     Raises:
         AssertionError: If any of the checks or verifications fail
     """
-    subnet_tempo = 50
+    subnet_tempo = 50 if subtensor.is_fast_blocks() else 10
     netuid = subtensor.get_total_subnets()  # 2
 
     # Wait for 2 tempos to pass as CR3 only reveals weights after 2 tempos
-    subtensor.wait_for_block(subnet_tempo * 2 + 1)
+    subtensor.wait_for_block(subtensor.block + (subnet_tempo * 2) + 1)
 
     print("Testing test_commit_and_reveal_weights")
     # Register root as Alice
@@ -267,16 +267,22 @@ def send_commit(salt_, weight_uids_, weight_vals_):
 
         send_commit(salt, weight_uids, weight_vals)
 
-        # let's wait for 3 (12 fast blocks) seconds between transactions
-        subtensor.wait_for_block(subtensor.block + 12)
+        # let's wait for 3 (12 fast blocks) seconds between transactions, next block for non-fast-blocks
+        waiting_block = (subtensor.block + 12) if subtensor.is_fast_blocks() else None
+        subtensor.wait_for_block(waiting_block)
 
     logging.console.info(
         f"[orange]Nonce after third commit_weights: "
         f"{subtensor.substrate.get_account_next_index(alice_wallet.hotkey.ss58_address)}[/orange]"
     )
 
     # Wait a few blocks
-    subtensor.wait_for_block(subtensor.block + subtensor.tempo(netuid) * 2)
+    waiting_block = (
+        (subtensor.block + subtensor.tempo(netuid) * 2)
+        if subtensor.is_fast_blocks()
+        else None
+    )
+    subtensor.wait_for_block(waiting_block)
 
     # Query the WeightCommits storage map for all three salts
     weight_commits = subtensor.query_module(

tests/e2e_tests/test_commitment.py

@@ -3,7 +3,10 @@
 
 from bittensor import logging
 from tests.e2e_tests.utils.chain_interactions import sudo_set_admin_utils
-from tests.e2e_tests.utils.e2e_test_utils import wait_to_start_call
+from tests.e2e_tests.utils.e2e_test_utils import (
+    wait_to_start_call,
+    async_wait_to_start_call,
+)
 
 logging.set_trace()
 
@@ -15,7 +18,7 @@ def test_commitment(local_chain, subtensor, alice_wallet, dave_wallet):
         "Subnet wasn't created successfully"
     )
 
-    assert wait_to_start_call(subtensor, dave_wallet, dave_subnet_netuid, 10)
+    assert wait_to_start_call(subtensor, dave_wallet, dave_subnet_netuid)
 
     with pytest.raises(SubstrateRequestException, match="AccountNotAllowedCommit"):
         subtensor.set_commitment(
@@ -93,11 +96,9 @@ async def test_commitment_async(
         "Subnet wasn't created successfully"
     )
 
-    await async_subtensor.wait_for_block(await async_subtensor.block + 20)
-    status, message = await async_subtensor.start_call(
-        dave_wallet, dave_subnet_netuid, True, True
+    assert await async_wait_to_start_call(
+        async_subtensor, dave_wallet, dave_subnet_netuid
     )
-    assert status, message
 
     async with async_subtensor as sub:
         with pytest.raises(SubstrateRequestException, match="AccountNotAllowedCommit"):

tests/e2e_tests/test_delegate.py

@@ -171,7 +171,7 @@ def test_change_take(local_chain, subtensor, alice_wallet, bob_wallet):
 
 
 @pytest.mark.asyncio
-async def test_delegates(subtensor, alice_wallet, bob_wallet):
+async def test_delegates(local_chain, subtensor, alice_wallet, bob_wallet):
     """
     Tests:
     - Check default Delegates
@@ -240,6 +240,7 @@ async def test_delegates(subtensor, alice_wallet, bob_wallet):
     assert subtensor.get_delegated(bob_wallet.coldkey.ss58_address) == []
 
     alice_subnet_netuid = subtensor.get_total_subnets()  # 2
+    set_tempo = 10
     # Register a subnet, netuid 2
     assert subtensor.register_subnet(alice_wallet), "Subnet wasn't created"
 
@@ -250,6 +251,17 @@ async def test_delegates(subtensor, alice_wallet, bob_wallet):
 
     assert wait_to_start_call(subtensor, alice_wallet, alice_subnet_netuid)
 
+    # set the same tempo for both type of nodes (fast and non-fast blocks)
+    assert (
+        sudo_set_admin_utils(
+            local_chain,
+            alice_wallet,
+            call_function="sudo_set_tempo",
+            call_params={"netuid": alice_subnet_netuid, "tempo": set_tempo},
+        )[0]
+        is True
+    )
+
     subtensor.add_stake(
         bob_wallet,
         alice_wallet.hotkey.ss58_address,
@@ -259,6 +271,9 @@ async def test_delegates(subtensor, alice_wallet, bob_wallet):
         wait_for_finalization=True,
     )
 
+    # let chain update validator_permits
+    subtensor.wait_for_block(subtensor.block + set_tempo + 1)
+
     bob_delegated = subtensor.get_delegated(bob_wallet.coldkey.ss58_address)
     assert bob_delegated == [
         DelegatedInfo(
@@ -272,9 +287,10 @@ async def test_delegates(subtensor, alice_wallet, bob_wallet):
                 bob_delegated[0].total_daily_return.rao
             ),
             netuid=alice_subnet_netuid,
-            stake=get_dynamic_balance(bob_delegated[0].stake.rao),
+            stake=get_dynamic_balance(bob_delegated[0].stake.rao, alice_subnet_netuid),
         ),
     ]
+    bittensor.logging.console.success("Test [green]test_delegates[/green] passed.")
 
 
 def test_nominator_min_required_stake(local_chain, subtensor, alice_wallet, bob_wallet):