From d0889dbd54f27b4e466add5db2bf0a8d8d5d2892 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 30 Jun 2025 20:29:24 +0200 Subject: [PATCH 1/7] Uses metadata v14 for events --- async_substrate_interface/async_substrate.py | 29 +++++++++++++------- async_substrate_interface/sync_substrate.py | 23 ++++++++++------ 2 files changed, 33 insertions(+), 19 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 421c8ab..e68a43d 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -999,7 +999,7 @@ async def decode_scale( else: if not runtime: runtime = await self.init_runtime(block_hash=block_hash) - if runtime.metadata_v15 is not None or force_legacy is True: + if runtime.metadata_v15 is not None and force_legacy is False: obj = decode_by_type_string(type_string, runtime.registry, scale_bytes) if self.decode_ss58: try: @@ -1930,7 +1930,13 @@ def convert_event_data(data): if key == "who": who = ss58_encode(bytes(value[0]), self.ss58_format) attributes["who"] = who - if isinstance(value, dict): + elif key == "from": + who_from = ss58_encode(bytes(value[0]), self.ss58_format) + attributes["from"] = who_from + elif key == "to": + who_to = ss58_encode(bytes(value[0]), self.ss58_format) + attributes["to"] = who_to + elif isinstance(value, dict): # Convert nested single-key dictionaries to their keys as strings for sub_key, sub_value in value.items(): if isinstance(sub_value, dict): @@ -1958,16 +1964,12 @@ def convert_event_data(data): block_hash = await self.get_chain_head() storage_obj = await self.query( - module="System", storage_function="Events", block_hash=block_hash + module="System", storage_function="Events", block_hash=block_hash, force_legacy_decode=True ) + # bt-decode Metadata V15 is not ideal for events. Force legacy decoding for this if storage_obj: for item in list(storage_obj): - try: - events.append(convert_event_data(item)) - except ( - AttributeError - ): # indicates this was legacy decoded with scalecodec - events.append(item) + events.append(item) return events async def get_metadata(self, block_hash=None) -> MetadataV15: @@ -2175,6 +2177,7 @@ async def _process_response( storage_item: Optional[ScaleType] = None, result_handler: Optional[ResultHandler] = None, runtime: Optional[Runtime] = None, + force_legacy_decode: bool = False ) -> tuple[Any, bool]: """ Processes the RPC call response by decoding it, returning it as is, or setting a handler for subscriptions, @@ -2187,6 +2190,7 @@ async def _process_response( storage_item: The ScaleType object used for decoding ScaleBytes results result_handler: the result handler coroutine used for handling longer-running subscriptions runtime: Optional Runtime to use for decoding. If not specified, the currently-loaded `self.runtime` is used + force_legacy_decode: Whether to force the use of the legacy Metadata V14 decoder Returns: (decoded response, completion) @@ -2208,7 +2212,7 @@ async def _process_response( q = bytes(query_value) else: q = query_value - result = await self.decode_scale(value_scale_type, q, runtime=runtime) + result = await self.decode_scale(value_scale_type, q, runtime=runtime, force_legacy=force_legacy_decode) if asyncio.iscoroutinefunction(result_handler): # For multipart responses as a result of subscriptions. message, bool_result = await result_handler(result, subscription_id) @@ -2223,6 +2227,7 @@ async def _make_rpc_request( result_handler: Optional[ResultHandler] = None, attempt: int = 1, runtime: Optional[Runtime] = None, + force_legacy_decode: bool = False ) -> RequestManager.RequestResults: request_manager = RequestManager(payloads) @@ -2267,6 +2272,7 @@ async def _make_rpc_request( storage_item, result_handler, runtime=runtime, + force_legacy_decode=force_legacy_decode ) request_manager.add_response( @@ -2298,6 +2304,7 @@ async def _make_rpc_request( storage_item, result_handler, attempt + 1, + force_legacy_decode ) return request_manager.get_results() @@ -3323,6 +3330,7 @@ async def query( subscription_handler=None, reuse_block_hash: bool = False, runtime: Optional[Runtime] = None, + force_legacy_decode: bool = False ) -> Optional[Union["ScaleObj", Any]]: """ Queries substrate. This should only be used when making a single request. For multiple requests, @@ -3355,6 +3363,7 @@ async def query( storage_item, result_handler=subscription_handler, runtime=runtime, + force_legacy_decode=force_legacy_decode ) result = responses[preprocessed.queryable][0] if isinstance(result, (list, tuple, int, float)): diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 92a01fa..80237b6 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -679,6 +679,7 @@ def decode_scale( type_string: str, scale_bytes: bytes, return_scale_obj=False, + force_legacy: bool = False, ) -> Union[ScaleObj, Any]: """ Helper function to decode arbitrary SCALE-bytes (e.g. 0x02000000) according to given RUST type_string @@ -689,6 +690,7 @@ def decode_scale( type_string: the type string of the SCALE object for decoding scale_bytes: the bytes representation of the SCALE object to decode return_scale_obj: Whether to return the decoded value wrapped in a SCALE-object-like wrapper, or raw. + force_legacy: Whether to force the use of the legacy Metadata V14 decoder Returns: Decoded object @@ -697,7 +699,7 @@ def decode_scale( # Decode AccountId bytes to SS58 address return ss58_encode(scale_bytes, self.ss58_format) else: - if self.runtime.metadata_v15 is not None: + if self.runtime.metadata_v15 is not None and force_legacy is False: obj = decode_by_type_string( type_string, self.runtime.registry, scale_bytes ) @@ -1631,16 +1633,12 @@ def convert_event_data(data): block_hash = self.get_chain_head() storage_obj = self.query( - module="System", storage_function="Events", block_hash=block_hash + module="System", storage_function="Events", block_hash=block_hash, force_legacy_decode=True ) + # bt-decode Metadata V15 is not ideal for events. Force legacy decoding for this if storage_obj: for item in list(storage_obj): - try: - events.append(convert_event_data(item)) - except ( - AttributeError - ): # indicates this was legacy decoded with scalecodec - events.append(item) + events.append(item) return events def get_metadata(self, block_hash=None) -> MetadataV15: @@ -1822,6 +1820,7 @@ def _process_response( value_scale_type: Optional[str] = None, storage_item: Optional[ScaleType] = None, result_handler: Optional[ResultHandler] = None, + force_legacy_decode: bool = False ) -> tuple[Any, bool]: """ Processes the RPC call response by decoding it, returning it as is, or setting a handler for subscriptions, @@ -1833,6 +1832,7 @@ def _process_response( value_scale_type: Scale Type string used for decoding ScaleBytes results storage_item: The ScaleType object used for decoding ScaleBytes results result_handler: the result handler coroutine used for handling longer-running subscriptions + force_legacy_decode: Whether to force legacy Metadata V14 decoding of the response Returns: (decoded response, completion) @@ -1854,7 +1854,7 @@ def _process_response( q = bytes(query_value) else: q = query_value - result = self.decode_scale(value_scale_type, q) + result = self.decode_scale(value_scale_type, q, force_legacy=force_legacy_decode) if isinstance(result_handler, Callable): # For multipart responses as a result of subscriptions. message, bool_result = result_handler(result, subscription_id) @@ -1868,6 +1868,7 @@ def _make_rpc_request( storage_item: Optional[ScaleType] = None, result_handler: Optional[ResultHandler] = None, attempt: int = 1, + force_legacy_decode: bool = False ) -> RequestManager.RequestResults: request_manager = RequestManager(payloads) _received = {} @@ -1901,6 +1902,7 @@ def _make_rpc_request( storage_item, result_handler, attempt + 1, + force_legacy_decode ) if "id" in response: _received[response["id"]] = response @@ -1932,6 +1934,7 @@ def _make_rpc_request( value_scale_type, storage_item, result_handler, + force_legacy_decode ) request_manager.add_response( item_id, decoded_response, complete @@ -2870,6 +2873,7 @@ def query( raw_storage_key: Optional[bytes] = None, subscription_handler=None, reuse_block_hash: bool = False, + force_legacy_decode: bool = False ) -> Optional[Union["ScaleObj", Any]]: """ Queries substrate. This should only be used when making a single request. For multiple requests, @@ -2895,6 +2899,7 @@ def query( value_scale_type, storage_item, result_handler=subscription_handler, + force_legacy_decode=force_legacy_decode ) result = responses[preprocessed.queryable][0] if isinstance(result, (list, tuple, int, float)): From 73a11fedaf8b28ddca8ff6e46e527e2883b81047 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 30 Jun 2025 20:40:34 +0200 Subject: [PATCH 2/7] Ensure ss58 format, if not set at init, is set for the runtime config as well as the Runtime. --- async_substrate_interface/async_substrate.py | 1 + async_substrate_interface/sync_substrate.py | 1 + 2 files changed, 2 insertions(+) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index e68a43d..88703cd 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -823,6 +823,7 @@ async def initialize(self): if ss58_prefix_constant: self.ss58_format = ss58_prefix_constant.value runtime.ss58_format = ss58_prefix_constant.value + runtime.runtime_config.ss58_format = ss58_prefix_constant.value self.initialized = True self._initializing = False diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 80237b6..0693e39 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -568,6 +568,7 @@ def initialize(self): if ss58_prefix_constant: self.ss58_format = ss58_prefix_constant.value self.runtime.ss58_format = ss58_prefix_constant.value + self.runtime.runtime_config.ss58_format = ss58_prefix_constant.value self.initialized = True def __exit__(self, exc_type, exc_val, exc_tb): From b98d4cbae4237695c0be25355b83c9776c1ece24 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 30 Jun 2025 20:40:43 +0200 Subject: [PATCH 3/7] Ruff --- async_substrate_interface/async_substrate.py | 21 ++++++++++++-------- async_substrate_interface/sync_substrate.py | 21 ++++++++++++-------- 2 files changed, 26 insertions(+), 16 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 88703cd..68ae0ff 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -1965,7 +1965,10 @@ def convert_event_data(data): block_hash = await self.get_chain_head() storage_obj = await self.query( - module="System", storage_function="Events", block_hash=block_hash, force_legacy_decode=True + module="System", + storage_function="Events", + block_hash=block_hash, + force_legacy_decode=True, ) # bt-decode Metadata V15 is not ideal for events. Force legacy decoding for this if storage_obj: @@ -2178,7 +2181,7 @@ async def _process_response( storage_item: Optional[ScaleType] = None, result_handler: Optional[ResultHandler] = None, runtime: Optional[Runtime] = None, - force_legacy_decode: bool = False + force_legacy_decode: bool = False, ) -> tuple[Any, bool]: """ Processes the RPC call response by decoding it, returning it as is, or setting a handler for subscriptions, @@ -2213,7 +2216,9 @@ async def _process_response( q = bytes(query_value) else: q = query_value - result = await self.decode_scale(value_scale_type, q, runtime=runtime, force_legacy=force_legacy_decode) + result = await self.decode_scale( + value_scale_type, q, runtime=runtime, force_legacy=force_legacy_decode + ) if asyncio.iscoroutinefunction(result_handler): # For multipart responses as a result of subscriptions. message, bool_result = await result_handler(result, subscription_id) @@ -2228,7 +2233,7 @@ async def _make_rpc_request( result_handler: Optional[ResultHandler] = None, attempt: int = 1, runtime: Optional[Runtime] = None, - force_legacy_decode: bool = False + force_legacy_decode: bool = False, ) -> RequestManager.RequestResults: request_manager = RequestManager(payloads) @@ -2273,7 +2278,7 @@ async def _make_rpc_request( storage_item, result_handler, runtime=runtime, - force_legacy_decode=force_legacy_decode + force_legacy_decode=force_legacy_decode, ) request_manager.add_response( @@ -2305,7 +2310,7 @@ async def _make_rpc_request( storage_item, result_handler, attempt + 1, - force_legacy_decode + force_legacy_decode, ) return request_manager.get_results() @@ -3331,7 +3336,7 @@ async def query( subscription_handler=None, reuse_block_hash: bool = False, runtime: Optional[Runtime] = None, - force_legacy_decode: bool = False + force_legacy_decode: bool = False, ) -> Optional[Union["ScaleObj", Any]]: """ Queries substrate. This should only be used when making a single request. For multiple requests, @@ -3364,7 +3369,7 @@ async def query( storage_item, result_handler=subscription_handler, runtime=runtime, - force_legacy_decode=force_legacy_decode + force_legacy_decode=force_legacy_decode, ) result = responses[preprocessed.queryable][0] if isinstance(result, (list, tuple, int, float)): diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 0693e39..9f6dd77 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -1634,7 +1634,10 @@ def convert_event_data(data): block_hash = self.get_chain_head() storage_obj = self.query( - module="System", storage_function="Events", block_hash=block_hash, force_legacy_decode=True + module="System", + storage_function="Events", + block_hash=block_hash, + force_legacy_decode=True, ) # bt-decode Metadata V15 is not ideal for events. Force legacy decoding for this if storage_obj: @@ -1821,7 +1824,7 @@ def _process_response( value_scale_type: Optional[str] = None, storage_item: Optional[ScaleType] = None, result_handler: Optional[ResultHandler] = None, - force_legacy_decode: bool = False + force_legacy_decode: bool = False, ) -> tuple[Any, bool]: """ Processes the RPC call response by decoding it, returning it as is, or setting a handler for subscriptions, @@ -1855,7 +1858,9 @@ def _process_response( q = bytes(query_value) else: q = query_value - result = self.decode_scale(value_scale_type, q, force_legacy=force_legacy_decode) + result = self.decode_scale( + value_scale_type, q, force_legacy=force_legacy_decode + ) if isinstance(result_handler, Callable): # For multipart responses as a result of subscriptions. message, bool_result = result_handler(result, subscription_id) @@ -1869,7 +1874,7 @@ def _make_rpc_request( storage_item: Optional[ScaleType] = None, result_handler: Optional[ResultHandler] = None, attempt: int = 1, - force_legacy_decode: bool = False + force_legacy_decode: bool = False, ) -> RequestManager.RequestResults: request_manager = RequestManager(payloads) _received = {} @@ -1903,7 +1908,7 @@ def _make_rpc_request( storage_item, result_handler, attempt + 1, - force_legacy_decode + force_legacy_decode, ) if "id" in response: _received[response["id"]] = response @@ -1935,7 +1940,7 @@ def _make_rpc_request( value_scale_type, storage_item, result_handler, - force_legacy_decode + force_legacy_decode, ) request_manager.add_response( item_id, decoded_response, complete @@ -2874,7 +2879,7 @@ def query( raw_storage_key: Optional[bytes] = None, subscription_handler=None, reuse_block_hash: bool = False, - force_legacy_decode: bool = False + force_legacy_decode: bool = False, ) -> Optional[Union["ScaleObj", Any]]: """ Queries substrate. This should only be used when making a single request. For multiple requests, @@ -2900,7 +2905,7 @@ def query( value_scale_type, storage_item, result_handler=subscription_handler, - force_legacy_decode=force_legacy_decode + force_legacy_decode=force_legacy_decode, ) result = responses[preprocessed.queryable][0] if isinstance(result, (list, tuple, int, float)): From 49d27a332f9532d392c8d5666e658714effad1c7 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 30 Jun 2025 20:40:51 +0200 Subject: [PATCH 4/7] Tests --- .../test_async_substrate_interface.py | 17 +++++++++++++++++ .../test_substrate_interface.py | 16 ++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/tests/integration_tests/test_async_substrate_interface.py b/tests/integration_tests/test_async_substrate_interface.py index 319330f..2c50213 100644 --- a/tests/integration_tests/test_async_substrate_interface.py +++ b/tests/integration_tests/test_async_substrate_interface.py @@ -115,3 +115,20 @@ async def test_fully_exhaust_query_map(): fully_exhausted_records_count += 1 assert fully_exhausted_records_count == initial_records_count_fully_exhaust assert initial_records_count_fully_exhaust == exhausted_records_count + + +@pytest.mark.asyncio +async def test_get_events_proper_decoding(): + # known block/hash pair that has the events we seek to decode + block = 5846788 + block_hash = "0x0a1c45063a59b934bfee827caa25385e60d5ec1fd8566a58b5cc4affc4eec412" + + async with AsyncSubstrateInterface(ARCHIVE_ENTRYPOINT) as substrate: + all_events = await substrate.get_events(block_hash=block_hash) + event = all_events[1] + print(type(event["attributes"])) + assert event["attributes"] == ( + "5G1NjW9YhXLadMWajvTkfcJy6up3yH2q1YzMXDTi6ijanChe", + 30, + "0xa6b4e5c8241d60ece0c25056b19f7d21ae845269fc771ad46bf3e011865129a5", + ) diff --git a/tests/integration_tests/test_substrate_interface.py b/tests/integration_tests/test_substrate_interface.py index 9710296..be4eb29 100644 --- a/tests/integration_tests/test_substrate_interface.py +++ b/tests/integration_tests/test_substrate_interface.py @@ -68,3 +68,19 @@ def test_ss58_conversion(): if len(value.value) > 0: for decoded_key in value.value: assert isinstance(decoded_key, str) + + +def test_get_events_proper_decoding(): + # known block/hash pair that has the events we seek to decode + block = 5846788 + block_hash = "0x0a1c45063a59b934bfee827caa25385e60d5ec1fd8566a58b5cc4affc4eec412" + + with SubstrateInterface(ARCHIVE_ENTRYPOINT) as substrate: + all_events = substrate.get_events(block_hash=block_hash) + event = all_events[1] + print(type(event["attributes"])) + assert event["attributes"] == ( + "5G1NjW9YhXLadMWajvTkfcJy6up3yH2q1YzMXDTi6ijanChe", + 30, + "0xa6b4e5c8241d60ece0c25056b19f7d21ae845269fc771ad46bf3e011865129a5", + ) From cc9cd09f6acee4828341b63198ee70ee1849a664 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 30 Jun 2025 21:04:07 +0200 Subject: [PATCH 5/7] Fix --- async_substrate_interface/async_substrate.py | 4 ++-- async_substrate_interface/sync_substrate.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 68ae0ff..42c7240 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -281,9 +281,9 @@ async def process_events(self): self.__weight = dispatch_info["weight"] if "Module" in dispatch_error: - module_index = dispatch_error["Module"][0]["index"] + module_index = dispatch_error["Module"]["index"] error_index = int.from_bytes( - bytes(dispatch_error["Module"][0]["error"]), + bytes(dispatch_error["Module"]["error"]), byteorder="little", signed=False, ) diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index 9f6dd77..da20af7 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -256,9 +256,9 @@ def process_events(self): self.__weight = dispatch_info["weight"] if "Module" in dispatch_error: - module_index = dispatch_error["Module"][0]["index"] + module_index = dispatch_error["Module"]["index"] error_index = int.from_bytes( - bytes(dispatch_error["Module"][0]["error"]), + bytes(dispatch_error["Module"]["error"]), byteorder="little", signed=False, ) From 7de0b38a27a31853b9b8bcb6bcc259e9f5b6a489 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 30 Jun 2025 21:09:31 +0200 Subject: [PATCH 6/7] Ensure extrinsic receipt events use the correct runtime metadata --- async_substrate_interface/async_substrate.py | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 42c7240..29f69bc 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -291,7 +291,15 @@ async def process_events(self): if isinstance(error_index, str): # Actual error index is first u8 in new [u8; 4] format error_index = int(error_index[2:4], 16) - module_error = self.substrate.metadata.get_module_error( + if self.block_hash: + runtime = await self.substrate.init_runtime( + block_hash=self.block_hash + ) + else: + runtime = await self.substrate.init_runtime( + block_id=self.block_number + ) + module_error = runtime.metadata.get_module_error( module_index=module_index, error_index=error_index ) self.__error_message = { From be318fd466ec72cea14bbcfebbb554e6a4207b82 Mon Sep 17 00:00:00 2001 From: Benjamin Himes Date: Mon, 30 Jun 2025 21:26:19 +0200 Subject: [PATCH 7/7] More backwards compat --- async_substrate_interface/async_substrate.py | 13 +++++++------ async_substrate_interface/sync_substrate.py | 13 +++++++------ 2 files changed, 14 insertions(+), 12 deletions(-) diff --git a/async_substrate_interface/async_substrate.py b/async_substrate_interface/async_substrate.py index 29f69bc..fb86216 100644 --- a/async_substrate_interface/async_substrate.py +++ b/async_substrate_interface/async_substrate.py @@ -281,16 +281,17 @@ async def process_events(self): self.__weight = dispatch_info["weight"] if "Module" in dispatch_error: - module_index = dispatch_error["Module"]["index"] - error_index = int.from_bytes( - bytes(dispatch_error["Module"]["error"]), - byteorder="little", - signed=False, - ) + if isinstance(dispatch_error["Module"], tuple): + module_index = dispatch_error["Module"][0] + error_index = dispatch_error["Module"][1] + else: + module_index = dispatch_error["Module"]["index"] + error_index = dispatch_error["Module"]["error"] if isinstance(error_index, str): # Actual error index is first u8 in new [u8; 4] format error_index = int(error_index[2:4], 16) + if self.block_hash: runtime = await self.substrate.init_runtime( block_hash=self.block_hash diff --git a/async_substrate_interface/sync_substrate.py b/async_substrate_interface/sync_substrate.py index da20af7..b7c4c15 100644 --- a/async_substrate_interface/sync_substrate.py +++ b/async_substrate_interface/sync_substrate.py @@ -256,16 +256,17 @@ def process_events(self): self.__weight = dispatch_info["weight"] if "Module" in dispatch_error: - module_index = dispatch_error["Module"]["index"] - error_index = int.from_bytes( - bytes(dispatch_error["Module"]["error"]), - byteorder="little", - signed=False, - ) + if isinstance(dispatch_error["Module"], tuple): + module_index = dispatch_error["Module"][0] + error_index = dispatch_error["Module"][1] + else: + module_index = dispatch_error["Module"]["index"] + error_index = dispatch_error["Module"]["error"] if isinstance(error_index, str): # Actual error index is first u8 in new [u8; 4] format error_index = int(error_index[2:4], 16) + module_error = self.substrate.metadata.get_module_error( module_index=module_index, error_index=error_index )