Fixes liquidity_list abysmal performance

Author: thewhaleking

bittensor/core/async_subtensor.py

@@ -10,6 +10,7 @@
 import scalecodec
 from async_substrate_interface import AsyncSubstrateInterface
 from async_substrate_interface.substrate_addons import RetryAsyncSubstrate
+from async_substrate_interface.utils.storage import StorageKey
 from bittensor_drand import get_encrypted_commitment
 from bittensor_wallet.utils import SS58_FORMAT
 from numpy.typing import NDArray
@@ -767,7 +768,8 @@ async def state_call(
         block_hash = await self.determine_block_hash(block, block_hash, reuse_block)
         return await self.substrate.rpc_request(
             method="state_call",
-            params=[method, data, block_hash] if block_hash else [method, data],
+            params=[method, data],
+            block_hash=block_hash,
             reuse_block_hash=reuse_block,
         )
 
@@ -2242,29 +2244,35 @@ async def get_liquidity_list(
             block=block, block_hash=block_hash, reuse_block=reuse_block
         )
 
-        query = self.substrate.query
+        # Fetch global fees and current price
+        fee_global_tao_query_sk = await self.substrate.create_storage_key(
+            pallet="Swap",
+            storage_function="FeeGlobalTao",
+            params=[netuid],
+            block_hash=block_hash,
+        )
+        fee_global_alpha_query_sk = await self.substrate.create_storage_key(
+            pallet="Swap",
+            storage_function="FeeGlobalAlpha",
+            params=[netuid],
+            block_hash=block_hash,
+        )
+        sqrt_price_query_sk = await self.substrate.create_storage_key(
+            pallet="Swap",
+            storage_function="AlphaSqrtPrice",
+            params=[netuid],
+            block_hash=block_hash,
+        )
         (
-            fee_global_tao,
-            fee_global_alpha,
-            sqrt_price,
+            (fee_global_tao_query, fee_global_alpha_query, sqrt_price_query),
             positions_response,
         ) = await asyncio.gather(
-            query(
-                module="Swap",
-                storage_function="FeeGlobalTao",
-                params=[netuid],
-                block_hash=block_hash,
-            ),
-            query(
-                module="Swap",
-                storage_function="FeeGlobalAlpha",
-                params=[netuid],
-                block_hash=block_hash,
-            ),
-            query(
-                module="Swap",
-                storage_function="AlphaSqrtPrice",
-                params=[netuid],
+            self.substrate.query_multi(
+                [
+                    fee_global_tao_query_sk,
+                    fee_global_alpha_query_sk,
+                    sqrt_price_query_sk,
+                ],
                 block_hash=block_hash,
             ),
             self.query_map(
@@ -2275,36 +2283,46 @@ async def get_liquidity_list(
             ),
         )
         # convert to floats
-        fee_global_tao = fixed_to_float(fee_global_tao)
-        fee_global_alpha = fixed_to_float(fee_global_alpha)
-        sqrt_price = fixed_to_float(sqrt_price)
+        fee_global_tao = fixed_to_float(fee_global_tao_query[1])
+        fee_global_alpha = fixed_to_float(fee_global_alpha_query[1])
+        sqrt_price = fixed_to_float(sqrt_price_query[1])
 
         # Fetch global fees and current price
         current_tick = price_to_tick(sqrt_price**2)
 
         # Fetch positions
-        positions = []
+        positions_values: list[tuple[dict, int, int]] = []
+        positions_storage_keys: list[StorageKey] = []
         async for _, p in positions_response:
             position = p.value
 
             tick_low_idx = position.get("tick_low")[0]
             tick_high_idx = position.get("tick_high")[0]
-
-            tick_low, tick_high = await asyncio.gather(
-                query(
-                    module="Swap",
-                    storage_function="Ticks",
-                    params=[netuid, tick_low_idx],
-                    block_hash=block_hash,
-                ),
-                query(
-                    module="Swap",
-                    storage_function="Ticks",
-                    params=[netuid, tick_high_idx],
-                    block_hash=block_hash,
-                ),
+            positions_values.append((position, tick_low_idx, tick_high_idx))
+            tick_low_sk = await self.substrate.create_storage_key(
+                pallet="Swap",
+                storage_function="Ticks",
+                params=[netuid, tick_low_idx],
+                block_hash=block_hash,
             )
+            tick_high_sk = await self.substrate.create_storage_key(
+                pallet="Swap",
+                storage_function="Ticks",
+                params=[netuid, tick_high_idx],
+                block_hash=block_hash,
+            )
+            positions_storage_keys.extend([tick_low_sk, tick_high_sk])
 
+        # query all our ticks at once
+        ticks_query = await self.substrate.query_multi(
+            positions_storage_keys, block_hash=block_hash
+        )
+        # iterator with just the values
+        ticks = iter([x[1] for x in ticks_query])
+        positions = []
+        for position, tick_low_idx, tick_high_idx in positions_values:
+            tick_low = next(ticks)
+            tick_high = next(ticks)
             # Calculate fees above/below range for both tokens
             tao_below = get_fees(
                 current_tick=current_tick,

bittensor/core/subtensor.py

@@ -429,9 +429,7 @@ def state_call(
         """
         block_hash = self.determine_block_hash(block)
         return self.substrate.rpc_request(
-            method="state_call",
-            params=[method, data],
-            block_hash=block_hash
+            method="state_call", params=[method, data], block_hash=block_hash
         )
 
     # Common subtensor calls ===========================================================================================
@@ -486,26 +484,22 @@ def blocks_since_last_step(
         )
         return query.value if query is not None and hasattr(query, "value") else query
 
-    def blocks_since_last_update(self, netuid: int, uid: int, block: Optional[int] = None) -> Optional[int]:
+    def blocks_since_last_update(self, netuid: int, uid: int) -> Optional[int]:
         """
         Returns the number of blocks since the last update for a specific UID in the subnetwork.
 
         Arguments:
             netuid (int): The unique identifier of the subnetwork.
             uid (int): The unique identifier of the neuron.
-            block (int): The block number for this query.
 
         Returns:
             Optional[int]: The number of blocks since the last update, or ``None`` if the subnetwork or UID does not
                 exist.
         """
-        call = self.get_hyperparameter(param_name="LastUpdate", netuid=netuid, block=block)
-        if not call:
-            return None
-        elif block is not None:
-            return block - int(call[uid])
-        else:
-            return self.get_current_block() - int(call[uid])
+        call = self.get_hyperparameter(
+            param_name="LastUpdate", netuid=netuid, block=block
+        )
+        return None if call is None else self.get_current_block() - int(call[uid])
 
     def bonds(
         self, netuid: int, block: Optional[int] = None
@@ -1504,17 +1498,33 @@ def get_liquidity_list(
 
         # Fetch global fees and current price
         fee_global_tao_query_sk = self.substrate.create_storage_key(
-            pallet="Swap", storage_function="FeeGlobalTao", params=[netuid], block_hash=block_hash
+            pallet="Swap",
+            storage_function="FeeGlobalTao",
+            params=[netuid],
+            block_hash=block_hash,
         )
         fee_global_alpha_query_sk = self.substrate.create_storage_key(
-            pallet="Swap", storage_function="FeeGlobalAlpha", params=[netuid], block_hash=block_hash
+            pallet="Swap",
+            storage_function="FeeGlobalAlpha",
+            params=[netuid],
+            block_hash=block_hash,
         )
         sqrt_price_query_sk = self.substrate.create_storage_key(
-            pallet="Swap", storage_function="AlphaSqrtPrice", params=[netuid], block_hash=block_hash
+            pallet="Swap",
+            storage_function="AlphaSqrtPrice",
+            params=[netuid],
+            block_hash=block_hash,
+        )
+        fee_global_tao_query, fee_global_alpha_query, sqrt_price_query = (
+            self.substrate.query_multi(
+                [
+                    fee_global_tao_query_sk,
+                    fee_global_alpha_query_sk,
+                    sqrt_price_query_sk,
+                ],
+                block_hash=block_hash,
+            )
         )
-        fee_global_tao_query, fee_global_alpha_query, sqrt_price_query = self.substrate.query_multi(
-            [fee_global_tao_query_sk, fee_global_alpha_query_sk, sqrt_price_query_sk],
-            block_hash=block_hash)
 
         fee_global_tao = fixed_to_float(fee_global_tao_query[1])
         fee_global_alpha = fixed_to_float(fee_global_alpha_query[1])
@@ -1546,12 +1556,14 @@ def get_liquidity_list(
                 pallet="Swap",
                 storage_function="Ticks",
                 params=[netuid, tick_high_idx],
-                block_hash=block_hash
+                block_hash=block_hash,
             )
             positions_values.append((position, tick_low_idx, tick_high_idx))
             positions_storage_keys.extend([tick_low_sk, tick_high_sk])
         # query all our ticks at once
-        ticks_query = self.substrate.query_multi(positions_storage_keys, block_hash=block_hash)
+        ticks_query = self.substrate.query_multi(
+            positions_storage_keys, block_hash=block_hash
+        )
         # iterator with just the values
         ticks = iter([x[1] for x in ticks_query])
         positions = []

pyproject.toml

@@ -36,7 +36,7 @@ dependencies = [
     "uvicorn",
     "bittensor-drand>=0.5.0",
     "bittensor-wallet>=3.1.0",
-    "async-substrate-interface>=1.3.1"
+    "async-substrate-interface>=1.4.1"
 ]
 
 [project.optional-dependencies]