Merge branch 'feat/thewhaleking/python-ss58-conversion' into feat/thewhaleking/fully-exhaust-query-map

Author: thewhaleking

async_substrate_interface/async_substrate.py

@@ -1378,9 +1378,9 @@ async def get_metadata_errors(
 
     async def get_metadata_error(
         self,
-        module_name,
-        error_name,
-        block_hash=None,
+        module_name: str,
+        error_name: str,
+        block_hash: Optional[str] = None,
         runtime: Optional[Runtime] = None,
     ):
         """
@@ -3113,9 +3113,9 @@ async def get_metadata_constants(self, block_hash=None) -> list[dict]:
 
     async def get_metadata_constant(
         self,
-        module_name,
-        constant_name,
-        block_hash=None,
+        module_name: str,
+        constant_name: str,
+        block_hash: Optional[str] = None,
         runtime: Optional[Runtime] = None,
     ):
         """

async_substrate_interface/types.py

@@ -649,8 +649,8 @@ def is_valid_ss58_address(self, value: str) -> bool:
     def serialize_storage_item(
         self,
         storage_item: ScaleType,
-        module,
-        spec_version_id,
+        module: str,
+        spec_version_id: int,
         runtime: Optional[Runtime] = None,
     ) -> dict:
         """

async_substrate_interface/utils/decoding.py

@@ -122,9 +122,11 @@ def concat_hash_len(key_hasher: str) -> int:
     middl_index = len(all_decoded) // 2
     decoded_keys = all_decoded[:middl_index]
     decoded_values = all_decoded[middl_index:]
-    for (kts, vts), (dk, dv) in zip(
-        zip(pre_decoded_key_types, pre_decoded_value_types),
-        zip(decoded_keys, decoded_values),
+    for kts, vts, dk, dv in zip(
+            pre_decoded_key_types,
+            pre_decoded_value_types,
+            decoded_keys,
+            decoded_values,
     ):
         try:
             # strip key_hashers to use as item key

tests/integration_tests/test_async_substrate_interface.py

@@ -0,0 +1,73 @@
+import pytest
+from scalecodec import ss58_encode
+
+from async_substrate_interface.async_substrate import AsyncSubstrateInterface
+from async_substrate_interface.types import ScaleObj
+from tests.helpers.settings import ARCHIVE_ENTRYPOINT, LATENT_LITE_ENTRYPOINT
+
+
+@pytest.mark.asyncio
+async def test_legacy_decoding():
+    # roughly 4000 blocks before metadata v15 was added
+    pre_metadata_v15_block = 3_010_611
+
+    async with AsyncSubstrateInterface(ARCHIVE_ENTRYPOINT) as substrate:
+        block_hash = await substrate.get_block_hash(pre_metadata_v15_block)
+        events = await substrate.get_events(block_hash)
+        assert isinstance(events, list)
+
+        query_map_result = await substrate.query_map(
+            module="SubtensorModule",
+            storage_function="NetworksAdded",
+            block_hash=block_hash,
+        )
+        async for key, value in query_map_result:
+            assert isinstance(key, int)
+            assert isinstance(value, ScaleObj)
+
+        timestamp = await substrate.query(
+            "Timestamp",
+            "Now",
+            block_hash=block_hash,
+        )
+        assert timestamp.value == 1716358476004
+
+
+@pytest.mark.asyncio
+async def test_ss58_conversion():
+    async with AsyncSubstrateInterface(
+        LATENT_LITE_ENTRYPOINT, ss58_format=42, decode_ss58=False
+    ) as substrate:
+        block_hash = await substrate.get_chain_finalised_head()
+        qm = await substrate.query_map(
+            "SubtensorModule",
+            "OwnedHotkeys",
+            block_hash=block_hash,
+        )
+        # only do the first page, bc otherwise this will be massive
+        for key, value in qm.records:
+            assert isinstance(key, tuple)
+            assert isinstance(value, ScaleObj)
+            assert isinstance(value.value, list)
+            assert len(key) == 1
+            for key_tuple in value.value:
+                assert len(key_tuple[0]) == 32
+                random_key = key_tuple[0]
+
+        ss58_of_key = ss58_encode(bytes(random_key), substrate.ss58_format)
+        assert isinstance(ss58_of_key, str)
+
+        substrate.decode_ss58 = True  # change to decoding True
+
+        qm = await substrate.query_map(
+            "SubtensorModule",
+            "OwnedHotkeys",
+            block_hash=block_hash,
+        )
+        for key, value in qm.records:
+            assert isinstance(key, str)
+            assert isinstance(value, ScaleObj)
+            assert isinstance(value.value, list)
+            if len(value.value) > 0:
+                for decoded_key in value.value:
+                    assert isinstance(decoded_key, str)

tests/integration_tests/test_substrate_interface.py

@@ -0,0 +1,70 @@
+from scalecodec import ss58_encode
+
+from async_substrate_interface.sync_substrate import SubstrateInterface
+from async_substrate_interface.types import ScaleObj
+from tests.helpers.settings import ARCHIVE_ENTRYPOINT, LATENT_LITE_ENTRYPOINT
+
+
+def test_legacy_decoding():
+    # roughly 4000 blocks before metadata v15 was added
+    pre_metadata_v15_block = 3_010_611
+
+    with SubstrateInterface(ARCHIVE_ENTRYPOINT) as substrate:
+        block_hash = substrate.get_block_hash(pre_metadata_v15_block)
+        events = substrate.get_events(block_hash)
+        assert isinstance(events, list)
+
+        query_map_result = substrate.query_map(
+            module="SubtensorModule",
+            storage_function="NetworksAdded",
+            block_hash=block_hash,
+        )
+        for key, value in query_map_result:
+            assert isinstance(key, int)
+            assert isinstance(value, ScaleObj)
+
+        timestamp = substrate.query(
+            "Timestamp",
+            "Now",
+            block_hash=block_hash,
+        )
+        assert timestamp.value == 1716358476004
+
+
+def test_ss58_conversion():
+    with SubstrateInterface(
+        LATENT_LITE_ENTRYPOINT, ss58_format=42, decode_ss58=False
+    ) as substrate:
+        block_hash = substrate.get_chain_finalised_head()
+        qm = substrate.query_map(
+            "SubtensorModule",
+            "OwnedHotkeys",
+            block_hash=block_hash,
+        )
+        # only do the first page, bc otherwise this will be massive
+        for key, value in qm.records:
+            assert isinstance(key, tuple)
+            assert isinstance(value, ScaleObj)
+            assert isinstance(value.value, list)
+            assert len(key) == 1
+            for key_tuple in value.value:
+                assert len(key_tuple[0]) == 32
+                random_key = key_tuple[0]
+
+        ss58_of_key = ss58_encode(bytes(random_key), substrate.ss58_format)
+        assert isinstance(ss58_of_key, str)
+
+        substrate.decode_ss58 = True  # change to decoding True
+
+        qm = substrate.query_map(
+            "SubtensorModule",
+            "OwnedHotkeys",
+            block_hash=block_hash,
+        )
+        for key, value in qm.records:
+            assert isinstance(key, str)
+            assert isinstance(value, ScaleObj)
+            assert isinstance(value.value, list)
+            if len(value.value) > 0:
+                for decoded_key in value.value:
+                    assert isinstance(decoded_key, str)

tests/unit_tests/asyncio_/test_substrate_interface.py

@@ -1,14 +1,11 @@
 import asyncio
-import time
 from unittest.mock import AsyncMock, MagicMock, ANY
 
 import pytest
-from scalecodec import ss58_encode
 from websockets.exceptions import InvalidURI
 
 from async_substrate_interface.async_substrate import AsyncSubstrateInterface
 from async_substrate_interface.types import ScaleObj
-from tests.helpers.settings import ARCHIVE_ENTRYPOINT, LATENT_LITE_ENTRYPOINT
 
 
 @pytest.mark.asyncio
@@ -102,7 +99,7 @@ async def test_runtime_call(monkeypatch):
 @pytest.mark.asyncio
 async def test_websocket_shutdown_timer():
     # using default ws shutdown timer of 5.0 seconds
-    async with AsyncSubstrateInterface(LATENT_LITE_ENTRYPOINT) as substrate:
+    async with AsyncSubstrateInterface("wss://lite.sub.latent.to:443") as substrate:
         await substrate.get_chain_head()
         await asyncio.sleep(6)
         assert (
@@ -116,112 +113,3 @@ async def test_websocket_shutdown_timer():
         await substrate.get_chain_head()
         await asyncio.sleep(6)  # same sleep time as before
         assert substrate.ws._initialized is True  # connection should still be open
-
-
-@pytest.mark.asyncio
-async def test_legacy_decoding():
-    # roughly 4000 blocks before metadata v15 was added
-    pre_metadata_v15_block = 3_010_611
-
-    async with AsyncSubstrateInterface(ARCHIVE_ENTRYPOINT) as substrate:
-        block_hash = await substrate.get_block_hash(pre_metadata_v15_block)
-        events = await substrate.get_events(block_hash)
-        assert isinstance(events, list)
-
-        query_map_result = await substrate.query_map(
-            module="SubtensorModule",
-            storage_function="NetworksAdded",
-            block_hash=block_hash,
-        )
-        async for key, value in query_map_result:
-            assert isinstance(key, int)
-            assert isinstance(value, ScaleObj)
-
-        timestamp = await substrate.query(
-            "Timestamp",
-            "Now",
-            block_hash=block_hash,
-        )
-        assert timestamp.value == 1716358476004
-
-
-@pytest.mark.asyncio
-async def test_ss58_conversion():
-    async with AsyncSubstrateInterface(
-        LATENT_LITE_ENTRYPOINT, ss58_format=42, decode_ss58=False
-    ) as substrate:
-        block_hash = await substrate.get_chain_finalised_head()
-        qm = await substrate.query_map(
-            "SubtensorModule",
-            "OwnedHotkeys",
-            block_hash=block_hash,
-        )
-        # only do the first page, bc otherwise this will be massive
-        for key, value in qm.records:
-            assert isinstance(key, tuple)
-            assert isinstance(value, ScaleObj)
-            assert isinstance(value.value, list)
-            assert len(key) == 1
-            for key_tuple in value.value:
-                assert len(key_tuple[0]) == 32
-                random_key = key_tuple[0]
-
-        ss58_of_key = ss58_encode(bytes(random_key), substrate.ss58_format)
-        assert isinstance(ss58_of_key, str)
-
-        substrate.decode_ss58 = True  # change to decoding True
-
-        qm = await substrate.query_map(
-            "SubtensorModule",
-            "OwnedHotkeys",
-            block_hash=block_hash,
-        )
-        for key, value in qm.records:
-            assert isinstance(key, str)
-            assert isinstance(value, ScaleObj)
-            assert isinstance(value.value, list)
-            if len(value.value) > 0:
-                for decoded_key in value.value:
-                    assert isinstance(decoded_key, str)
-
-
-@pytest.mark.asyncio
-async def test_fully_exhaust_query_map():
-    async with AsyncSubstrateInterface(LATENT_LITE_ENTRYPOINT) as substrate:
-        block_hash = await substrate.get_chain_finalised_head()
-        non_fully_exhauster_start = time.time()
-        non_fully_exhausted_qm = await substrate.query_map(
-            "SubtensorModule",
-            "CRV3WeightCommits",
-            block_hash=block_hash,
-        )
-        initial_records_count = len(non_fully_exhausted_qm.records)
-        assert initial_records_count <= 100  # default page size
-        exhausted_records_count = 0
-        async for _ in non_fully_exhausted_qm:
-            exhausted_records_count += 1
-        non_fully_exhausted_time = time.time() - non_fully_exhauster_start
-
-        assert len(non_fully_exhausted_qm.records) >= initial_records_count
-        fully_exhausted_start = time.time()
-        fully_exhausted_qm = await substrate.query_map(
-            "SubtensorModule",
-            "CRV3WeightCommits",
-            block_hash=block_hash,
-            fully_exhaust=True,
-        )
-
-        fully_exhausted_time = time.time() - fully_exhausted_start
-        initial_records_count_fully_exhaust = len(fully_exhausted_qm.records)
-        assert fully_exhausted_time <= non_fully_exhausted_time, (
-            f"Fully exhausted took longer than non-fully exhausted with "
-            f"{len(non_fully_exhausted_qm.records)} records in non-fully exhausted "
-            f"in {non_fully_exhausted_time} seconds, and {initial_records_count_fully_exhaust} in fully exhausted"
-            f" in {fully_exhausted_time} seconds. This could be caused by the fact that on this specific block, "
-            f"there are fewer records than take up a single page. This difference should still be small."
-        )
-        fully_exhausted_records_count = 0
-        async for _ in fully_exhausted_qm:
-            fully_exhausted_records_count += 1
-        assert fully_exhausted_records_count == initial_records_count_fully_exhaust
-        assert initial_records_count_fully_exhaust == exhausted_records_count