Merge pull request #55 from opentensor/release/1.0.2

ibraheem-abe · web-flow · commit acb24ddffa5f · 2025-02-19T16:44:51.000-08:00
Release/1.0.2
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,5 +1,23 @@
 # Changelog
 
+## 1.0.2 /2025-02-19
+
+## What's Changed
+* Closes the connection on the object being garbage-collected by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/51
+* Generate UIDs for websockets by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/50
+* Dynamically pulls the info for Vec<AccountId> from the metadata by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/47
+* Fix readme by @igorsyl in https://github.com/opentensor/async-substrate-interface/pull/46
+* Handle options with bt-decode by @thewhaleking in https://github.com/opentensor/async-substrate-interface/pull/52
+* Backmerge main to staging 101 by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/53
+* Handles None change_data by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/54
+
+## New Contributors
+* @igorsyl made their first contribution in https://github.com/opentensor/async-substrate-interface/pull/46
+
+**Full Changelog**: https://github.com/opentensor/async-substrate-interface/compare/v1.0.1...v1.0.2
+
+## 1.0.1 /2025-02-17
+
 ## What's Changed
 * Updates type for vec acc id by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/45
 * Backmerge main staging 101 by @ibraheem-opentensor in https://github.com/opentensor/async-substrate-interface/pull/48
diff --git a/README.md b/README.md
@@ -1,3 +1,67 @@
-This is a modernised version of the py-substrate-interface library, with the ability to use it asynchronously (as well as synchronously). It aims to be almost fully API-compatible with the original library.
+# Async Substrate Interface
+This project provides an asynchronous interface for interacting with [Substrate](https://substrate.io/)-based blockchains. It is based on the [py-substrate-interface](https://github.com/polkascan/py-substrate-interface) project.
 
-In addition to it's async nature, it is additionally improved with using bt-decode rather than py-scale-codec for significantly faster SCALE decoding.
+Additionally, this project uses [bt-decode](https://github.com/opentensor/bt-decode) instead of [py-scale-codec](https://github.com/polkascan/py-scale-codec) for faster [SCALE](https://docs.substrate.io/reference/scale-codec/) decoding.
+
+## Installation
+
+To install the package, use the following command:
+
+```bash
+pip install async-substrate-interface
+```
+
+## Usage
+
+Here are examples of how to use the sync and async inferfaces:
+
+```python
+from async_substrate_interface import SubstrateInterface
+
+def main():
+    substrate = SubstrateInterface(
+        url="wss://rpc.polkadot.io"
+    )
+    with substrate:
+        result = substrate.query(
+            module='System',
+            storage_function='Account',
+            params=['5CZs3T15Ky4jch1sUpSFwkUbYEnsCfe1WCY51fH3SPV6NFnf']
+        )
+
+        print(result)
+
+main()
+```
+
+```python
+import asyncio
+from async_substrate_interface import AsyncSubstrateInterface
+
+async def main():
+    substrate = AsyncSubstrateInterface(
+        url="wss://rpc.polkadot.io"
+    )
+    async with substrate:
+        result = await substrate.query(
+            module='System',
+            storage_function='Account',
+            params=['5CZs3T15Ky4jch1sUpSFwkUbYEnsCfe1WCY51fH3SPV6NFnf']
+        )
+
+        print(result)
+
+asyncio.run(main())
+```
+
+## Contributing
+
+Contributions are welcome! Please open an issue or submit a pull request to the `staging` branch.
+
+## License
+
+This project is licensed under the MIT License. See the [LICENSE](LICENSE) file for details.
+
+## Contact
+
+For any questions or inquiries, please join the Bittensor Development Discord server: [Church of Rao](https://discord.gg/XC7ucQmq2Q).
diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py
@@ -48,7 +48,7 @@
     SubstrateMixin,
     Preprocessed,
 )
-from async_substrate_interface.utils import hex_to_bytes, json
+from async_substrate_interface.utils import hex_to_bytes, json, generate_unique_id
 from async_substrate_interface.utils.decoding import (
     _determine_if_old_runtime_call,
     _bt_decode_to_dict_or_list,
@@ -507,7 +507,6 @@ def __init__(
         # TODO reconnection logic
         self.ws_url = ws_url
         self.ws: Optional["ClientConnection"] = None
-        self.id = 0
         self.max_subscriptions = max_subscriptions
         self.max_connections = max_connections
         self.shutdown_timer = shutdown_timer
@@ -543,8 +542,6 @@ async def connect(self, force=False):
                 connect(self.ws_url, **self._options), timeout=10
             )
             self._receiving_task = asyncio.create_task(self._start_receiving())
-        if force:
-            self.id = 100
 
     async def __aexit__(self, exc_type, exc_val, exc_tb):
         async with self._lock:  # TODO is this actually what I want to happen?
@@ -556,7 +553,6 @@ async def __aexit__(self, exc_type, exc_val, exc_tb):
                 except asyncio.CancelledError:
                     pass
             if self._in_use == 0 and self.ws is not None:
-                self.id = 0
                 self._open_subscriptions = 0
                 self._exit_task = asyncio.create_task(self._exit_with_timer())
 
@@ -582,7 +578,6 @@ async def shutdown(self):
             self.ws = None
             self._initialized = False
             self._receiving_task = None
-            self.id = 0
 
     async def _recv(self) -> None:
         try:
@@ -625,8 +620,7 @@ async def send(self, payload: dict) -> int:
             id: the internal ID of the request (incremented int)
         """
         # async with self._lock:
-        original_id = self.id
-        self.id += 1
+        original_id = generate_unique_id(json.dumps(payload))
         # self._open_subscriptions += 1
         try:
             await self.ws.send(json.dumps({**payload, **{"id": original_id}}))
@@ -719,6 +713,8 @@ def __init__(
         self.metadata_version_hex = "0x0f000000"  # v15
         self.reload_type_registry()
         self._initializing = False
+        self.registry_type_map = {}
+        self.type_id_to_name = {}
 
     async def __aenter__(self):
         await self.initialize()
@@ -735,8 +731,9 @@ async def initialize(self):
                     chain = await self.rpc_request("system_chain", [])
                     self._chain = chain.get("result")
                 init_load = await asyncio.gather(
-                    self.load_registry(), self._first_initialize_runtime(),
-                    return_exceptions=True
+                    self.load_registry(),
+                    self._first_initialize_runtime(),
+                    return_exceptions=True,
                 )
                 for potential_exception in init_load:
                     if isinstance(potential_exception, Exception):
@@ -812,6 +809,7 @@ async def load_registry(self):
             metadata_option_bytes
         )
         self.registry = PortableRegistry.from_metadata_v15(self.metadata_v15)
+        self._load_registry_type_map()
 
     async def _load_registry_at_block(self, block_hash: str) -> MetadataV15:
         # Should be called for any block that fails decoding.
@@ -894,15 +892,14 @@ async def decode_scale(
         Returns:
             Decoded object
         """
-        if scale_bytes == b"\x00":
-            obj = None
+        if scale_bytes == b"":
+            return None
+        if type_string == "scale_info::0":  # Is an AccountId
+            # Decode AccountId bytes to SS58 address
+            return ss58_encode(scale_bytes, SS58_FORMAT)
         else:
-            if type_string == "scale_info::0":  # Is an AccountId
-                # Decode AccountId bytes to SS58 address
-                return ss58_encode(scale_bytes, SS58_FORMAT)
-            else:
-                await self._wait_for_registry(_attempt, _retries)
-                obj = decode_by_type_string(type_string, self.registry, scale_bytes)
+            await self._wait_for_registry(_attempt, _retries)
+            obj = decode_by_type_string(type_string, self.registry, scale_bytes)
         if return_scale_obj:
             return ScaleObj(obj)
         else:
@@ -2235,7 +2232,7 @@ async def query_multi(
                 # Decode result for specified storage_key
                 storage_key = storage_key_map[change_storage_key]
                 if change_data is None:
-                    change_data = b"\x00"
+                    change_data = b""
                 else:
                     change_data = bytes.fromhex(change_data[2:])
                 result.append(
diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py
@@ -30,7 +30,7 @@
     Preprocessed,
     ScaleObj,
 )
-from async_substrate_interface.utils import hex_to_bytes, json
+from async_substrate_interface.utils import hex_to_bytes, json, generate_unique_id
 from async_substrate_interface.utils.decoding import (
     _determine_if_old_runtime_call,
     _bt_decode_to_dict_or_list,
@@ -518,13 +518,18 @@ def __init__(
         self.metadata_version_hex = "0x0f000000"  # v15
         self.reload_type_registry()
         self.ws = self.connect(init=True)
+        self.registry_type_map = {}
+        self.type_id_to_name = {}
         if not _mock:
             self.initialize()
 
     def __enter__(self):
         self.initialize()
         return self
 
+    def __del__(self):
+        self.close()
+
     def initialize(self):
         """
         Initialize the connection to the chain.
@@ -612,6 +617,7 @@ def load_registry(self):
             metadata_option_bytes
         )
         self.registry = PortableRegistry.from_metadata_v15(self.metadata_v15)
+        self._load_registry_type_map()
 
     def _load_registry_at_block(self, block_hash: str) -> MetadataV15:
         # Should be called for any block that fails decoding.
@@ -646,15 +652,11 @@ def decode_scale(
         Returns:
             Decoded object
         """
-
-        if scale_bytes == b"\x00":
-            obj = None
+        if type_string == "scale_info::0":  # Is an AccountId
+            # Decode AccountId bytes to SS58 address
+            return ss58_encode(scale_bytes, SS58_FORMAT)
         else:
-            if type_string == "scale_info::0":  # Is an AccountId
-                # Decode AccountId bytes to SS58 address
-                return ss58_encode(scale_bytes, SS58_FORMAT)
-            else:
-                obj = decode_by_type_string(type_string, self.registry, scale_bytes)
+            obj = decode_by_type_string(type_string, self.registry, scale_bytes)
         if return_scale_obj:
             return ScaleObj(obj)
         else:
@@ -1681,9 +1683,9 @@ def _make_rpc_request(
         subscription_added = False
 
         ws = self.connect(init=False if attempt == 1 else True)
-        item_id = 0
         for payload in payloads:
-            item_id += 1
+            payload_str = json.dumps(payload["payload"])
+            item_id = generate_unique_id(payload_str)
             ws.send(json.dumps({**payload["payload"], **{"id": item_id}}))
             request_manager.add_request(item_id, payload["id"])
 
diff --git a/async_substrate_interface/types.py b/async_substrate_interface/types.py
@@ -13,6 +13,8 @@
 from scalecodec.type_registry import load_type_registry_preset
 from scalecodec.types import GenericCall, ScaleType
 
+from .utils import json
+
 
 logger = logging.getLogger("async_substrate_interface")
 
@@ -349,6 +351,9 @@ class SubstrateMixin(ABC):
     type_registry: Optional[dict]
     ss58_format: Optional[int]
     ws_max_size = 2**32
+    registry_type_map: dict[str, int]
+    type_id_to_name: dict[int, str]
+    metadata_v15 = None
 
     @property
     def chain(self):
@@ -604,6 +609,70 @@ def serialize_module_error(module, error, spec_version) -> dict:
             "spec_version": spec_version,
         }
 
+    def _load_registry_type_map(self):
+        registry_type_map = {}
+        type_id_to_name = {}
+        types = json.loads(self.registry.registry)["types"]
+        for type_entry in types:
+            type_type = type_entry["type"]
+            type_id = type_entry["id"]
+            type_def = type_type["def"]
+            type_path = type_type.get("path")
+            if type_entry.get("params") or type_def.get("variant"):
+                continue  # has generics or is Enum
+            if type_path:
+                type_name = type_path[-1]
+                registry_type_map[type_name] = type_id
+                type_id_to_name[type_id] = type_name
+            else:
+                # probably primitive
+                if type_def.get("primitive"):
+                    type_name = type_def["primitive"]
+                    registry_type_map[type_name] = type_id
+                    type_id_to_name[type_id] = type_name
+        for type_entry in types:
+            type_type = type_entry["type"]
+            type_id = type_entry["id"]
+            type_def = type_type["def"]
+            if type_def.get("sequence"):
+                sequence_type_id = type_def["sequence"]["type"]
+                inner_type = type_id_to_name.get(sequence_type_id)
+                if inner_type:
+                    type_name = f"Vec<{inner_type}>"
+                    type_id_to_name[type_id] = type_name
+                    registry_type_map[type_name] = type_id
+            elif type_def.get("array"):
+                array_type_id = type_def["array"]["type"]
+                inner_type = type_id_to_name.get(array_type_id)
+                maybe_len = type_def["array"].get("len")
+                if inner_type:
+                    if maybe_len:
+                        type_name = f"[{inner_type}; {maybe_len}]"
+                    else:
+                        type_name = f"[{inner_type}]"
+                    type_id_to_name[type_id] = type_name
+                    registry_type_map[type_name] = type_id
+            elif type_def.get("compact"):
+                compact_type_id = type_def["compact"]["type"]
+                inner_type = type_id_to_name.get(compact_type_id)
+                if inner_type:
+                    type_name = f"Compact<{inner_type}>"
+                    type_id_to_name[type_id] = type_name
+                    registry_type_map[type_name] = type_id
+            elif type_def.get("tuple"):
+                tuple_type_ids = type_def["tuple"]
+                type_names = []
+                for inner_type_id in tuple_type_ids:
+                    inner_type = type_id_to_name.get(inner_type_id)
+                    if inner_type:
+                        type_names.append(inner_type)
+                type_name = ", ".join(type_names)
+                type_name = f"({type_name})"
+                type_id_to_name[type_id] = type_name
+                registry_type_map[type_name] = type_id
+        self.registry_type_map = registry_type_map
+        self.type_id_to_name = type_id_to_name
+
     def reload_type_registry(
         self, use_remote_preset: bool = True, auto_discover: bool = True
     ):
@@ -726,12 +795,19 @@ def _encode_scale(self, type_string, value: Any) -> bytes:
         if value is None:
             result = b"\x00"
         else:
+            try:
+                vec_acct_id = (
+                    f"scale_info::{self.registry_type_map['Vec<AccountId32>']}"
+                )
+            except KeyError:
+                vec_acct_id = "scale_info::152"
+
             if type_string == "scale_info::0":  # Is an AccountId
                 # encode string into AccountId
                 ## AccountId is a composite type with one, unnamed field
                 return bytes.fromhex(ss58_decode(value, SS58_FORMAT))
 
-            elif type_string == "scale_info::152":  # Vec<AccountId>
+            elif type_string == vec_acct_id:  # Vec<AccountId>
                 if not isinstance(value, (list, tuple)):
                     value = [value]
 
diff --git a/async_substrate_interface/utils/__init__.py b/async_substrate_interface/utils/__init__.py
@@ -1,4 +1,10 @@
 import importlib
+import hashlib
+
+
+def generate_unique_id(item: str, length=10):
+    hashed_value = hashlib.sha256(item.encode()).hexdigest()
+    return hashed_value[:length]
 
 
 def hex_to_bytes(hex_str: str) -> bytes:
diff --git a/pyproject.toml b/pyproject.toml
