v0.50.0: Integrate frame-decode, redo storage APIs and break up Error. (#2100)

* WIP integrating new frame-decode and working out new storage APIS

* WIP: first pass adding new storage things to subxt-core

* Second pass over Address type and start impl in Subxt

* WIP new storage APIs

* WIP New storage APIs roughly completed, lots of errors still

* Remove PlainorMap enum; plain and map values now use same struct to simplify usage

* Begin 'fixing' errors

* WIP splitting errors and tidying payload/address traits

* Get subxt-core compiling

* Small fixes in subxt-core and remove metadata mod

* subxt-core: cargo check --all-targets passes

* Fix test

* WIP starting to update subxt from subxt-core changes

* WIP splitting up subxt errors into smaller variants

* WIP errors: add DispatchError errors

* Port new Storage APIs to subxt-core

* cargo check -p subxt passes

* Quick-fix errors in subxt-cli (explore subcommand)

* fmt

* Finish fixing codegen up and start fixing examples

* get Subxt examples compiling and bytes_at for constants

* Add some arcs to limit lifetimes in subxt/subxt-core storage APIs

* A little Arcing to allow more method chaining in Storage APIs, aligning with Subxt

* Update codegen test

* cargo check --all-targets passing

* cargo check --features 'unstable-light-client' passing

* clippy

* Remove unused dep in subxt

* use published frame-decode

* fix wasm-example

* Add new tx extension to fix daily tests

* Remove unused subxt_core::dynamic::DecodedValue type

* Update book to match changes

* Update docs to fix more broken bits

* Add missing docs

* fmt

* allow larger result errs for now

* Add missing alloc imports in subxt-core

* Fix doc tests and fix bug getting constant info

* Fix V14 -> Metadata transform for storage & constants

* Fix parachain example

* Fix FFI example

* BlockLength decodes t ostruct, not u128

* use fetch/iter shorthands rather than entry in most storage tests

* Fix some integration tests

* Fix Runtime codegen tests

* Expose the dynamic custom_value selecter and use in a UI test

* Update codegen metadata

* Tidy CLI storage query and support (str,str) as a storage address

* Add (str,str) as valid constant address too

* Show string tuple in constants example

* Via the magic of traits, avoid needing any clones of queries/addresses and accept references to them

* clippy

Author: jsdw

Cargo.lock

@@ -81,7 +81,7 @@ darling = "0.20.10"
 derive-where = "1.2.7"
 either = { version = "1.13.0", default-features = false }
 finito = { version = "0.1.0", default-features = false }
-frame-decode = { version = "0.10.0", default-features = false }
+frame-decode = { version = "0.11.1", default-features = false }
 frame-metadata = { version = "23.0.0", default-features = false }
 futures = { version = "0.3.31", default-features = false, features = ["std"] }
 getrandom = { version = "0.2", default-features = false }
@@ -98,12 +98,12 @@ proc-macro2 = "1.0.86"
 quote = "1.0.37"
 regex = { version = "1.11.0", default-features = false }
 scale-info = { version = "2.11.4", default-features = false }
-scale-value = { version = "0.18.0", default-features = false }
+scale-value = { version = "0.18.1", default-features = false }
 scale-bits = { version = "0.7.0", default-features = false }
 scale-decode = { version = "0.16.0", default-features = false }
 scale-encode = { version = "0.10.0", default-features = false }
 scale-type-resolver = { version = "0.2.0" }
-scale-info-legacy = { version = "0.2.3" }
+scale-info-legacy = { version = "0.2.4" }
 scale-typegen = "0.11.1"
 scale-typegen-description = "0.11.0"
 serde = { version = "1.0.210", default-features = false, features = ["derive"] }

Cargo.toml

@@ -215,7 +215,6 @@ struct StorageEntryDiff {
     key_different: bool,
     value_different: bool,
     default_different: bool,
-    modifier_different: bool,
 }
 
 impl StorageEntryDiff {
@@ -225,41 +224,32 @@ impl StorageEntryDiff {
         metadata_1: &Metadata,
         metadata_2: &Metadata,
     ) -> Self {
-        let value_1_ty_id = storage_entry_1.entry_type().value_ty();
+        let value_1_ty_id = storage_entry_1.value_ty();
         let value_1_hash = metadata_1
             .type_hash(value_1_ty_id)
             .expect("type is in metadata; qed");
-        let value_2_ty_id = storage_entry_2.entry_type().value_ty();
+        let value_2_ty_id = storage_entry_2.value_ty();
         let value_2_hash = metadata_2
             .type_hash(value_2_ty_id)
             .expect("type is in metadata; qed");
         let value_different = value_1_hash != value_2_hash;
 
-        let key_1_hash = storage_entry_1
-            .entry_type()
-            .key_ty()
-            .map(|key_ty| {
-                metadata_1
-                    .type_hash(key_ty)
-                    .expect("type is in metadata; qed")
-            })
-            .unwrap_or_default();
-        let key_2_hash = storage_entry_2
-            .entry_type()
-            .key_ty()
-            .map(|key_ty| {
-                metadata_2
-                    .type_hash(key_ty)
-                    .expect("type is in metadata; qed")
-            })
-            .unwrap_or_default();
-        let key_different = key_1_hash != key_2_hash;
+        let key_parts_same = storage_entry_1.keys().len() == storage_entry_2.keys().len()
+            && storage_entry_1
+                .keys()
+                .zip(storage_entry_2.keys())
+                .all(|(a, b)| {
+                    let a_hash = metadata_1.type_hash(a.key_id).expect("type is in metadata");
+                    let b_hash = metadata_2.type_hash(b.key_id).expect("type is in metadata");
+                    a.hasher == b.hasher && a_hash == b_hash
+                });
+
+        let key_different = !key_parts_same;
 
         StorageEntryDiff {
             key_different,
             value_different,
-            default_different: storage_entry_1.default_bytes() != storage_entry_2.default_bytes(),
-            modifier_different: storage_entry_1.modifier() != storage_entry_2.modifier(),
+            default_different: storage_entry_1.default_value() != storage_entry_2.default_value(),
         }
     }
 
@@ -271,9 +261,6 @@ impl StorageEntryDiff {
         if self.value_different {
             strings.push("value type");
         }
-        if self.modifier_different {
-            strings.push("modifier");
-        }
         if self.default_different {
             strings.push("default value");
         }

cli/src/commands/diff.rs

@@ -12,7 +12,7 @@ use subxt::utils::H256;
 use subxt::{
     OfflineClient,
     config::SubstrateConfig,
-    metadata::{Metadata, types::PalletMetadata},
+    metadata::{Metadata, PalletMetadata},
 };
 
 use crate::utils::{

cli/src/commands/explore/pallets/calls.rs

@@ -2,7 +2,7 @@ use clap::Args;
 use color_eyre::eyre::eyre;
 use indoc::{formatdoc, writedoc};
 use scale_typegen_description::type_description;
-use subxt::metadata::{Metadata, types::PalletMetadata};
+use subxt::metadata::{Metadata, PalletMetadata};
 
 use crate::utils::{Indent, SyntaxHighlight, first_paragraph_of_docs, format_scale_value};
 

cli/src/commands/explore/pallets/constants.rs

@@ -2,7 +2,7 @@ use clap::Args;
 use color_eyre::eyre::eyre;
 use indoc::{formatdoc, writedoc};
 use scale_info::{Variant, form::PortableForm};
-use subxt::metadata::{Metadata, types::PalletMetadata};
+use subxt::metadata::{Metadata, PalletMetadata};
 
 use crate::utils::{Indent, fields_description, first_paragraph_of_docs};
 

cli/src/commands/explore/pallets/events.rs

@@ -1,17 +1,11 @@
 use clap::Args;
-use color_eyre::{
-    eyre::{bail, eyre},
-    owo_colors::OwoColorize,
-};
+use color_eyre::{eyre::bail, owo_colors::OwoColorize};
 use indoc::{formatdoc, writedoc};
 use scale_typegen_description::type_description;
 use scale_value::Value;
 use std::fmt::Write;
 use std::write;
-use subxt::metadata::{
-    Metadata,
-    types::{PalletMetadata, StorageEntryType, StorageMetadata},
-};
+use subxt::metadata::{Metadata, PalletMetadata, StorageMetadata};
 
 use crate::utils::{
     FileOrUrl, Indent, SyntaxHighlight, create_client, first_paragraph_of_docs,
@@ -75,12 +69,7 @@ pub async fn explore_storage(
         );
     };
 
-    let (return_ty_id, key_ty_id) = match storage.entry_type() {
-        StorageEntryType::Plain(value) => (*value, None),
-        StorageEntryType::Map {
-            value_ty, key_ty, ..
-        } => (*value_ty, Some(*key_ty)),
-    };
+    let return_ty_id = storage.value_ty();
 
     let key_value_placeholder = "<KEY_VALUE>".blue();
 
@@ -114,15 +103,30 @@ pub async fn explore_storage(
     "}?;
 
     // inform user about shape of the key if it can be provided:
-    if let Some(key_ty_id) = key_ty_id {
-        let key_ty_description = type_description(key_ty_id, metadata.types(), true)
-            .expect("No type Description")
-            .indent(4)
-            .highlight();
+    let storage_keys = storage.keys().collect::<Vec<_>>();
+    if !storage_keys.is_empty() {
+        let key_ty_description = format!(
+            "({})",
+            storage_keys
+                .iter()
+                .map(|key| type_description(key.key_id, metadata.types(), true)
+                    .expect("No type Description"))
+                .collect::<Vec<_>>()
+                .join(", ")
+        )
+        .indent(4)
+        .highlight();
 
-        let key_ty_example = type_example(key_ty_id, metadata.types())
-            .indent(4)
-            .highlight();
+        let key_ty_example = format!(
+            "({})",
+            storage_keys
+                .iter()
+                .map(|key| type_example(key.key_id, metadata.types()).to_string())
+                .collect::<Vec<_>>()
+                .join(", ")
+        )
+        .indent(4)
+        .highlight();
 
         writedoc! {output, "
 
@@ -144,7 +148,8 @@ pub async fn explore_storage(
         return Ok(());
     }
 
-    let storage_entry_keys: Vec<Value> = match (!trailing_args.is_empty(), key_ty_id.is_some()) {
+    let storage_entry_keys: Vec<Value> = match (!trailing_args.is_empty(), !storage_keys.is_empty())
+    {
         // keys provided, keys not needed.
         (true, false) => {
             let trailing_args_str = trailing_args.join(" ");
@@ -190,18 +195,17 @@ pub async fn explore_storage(
     // construct the client:
     let client = create_client(&file_or_url).await?;
 
-    let storage_query = subxt::dynamic::storage(pallet_name, storage.name(), storage_entry_keys);
-    let decoded_value_thunk_or_none = client
+    // Fetch the value:
+    let storage_value = client
         .storage()
         .at_latest()
         .await?
-        .fetch(&storage_query)
-        .await?;
+        .fetch((pallet_name, storage.name()), storage_entry_keys)
+        .await?
+        .decode()?;
 
-    let decoded_value_thunk =
-        decoded_value_thunk_or_none.ok_or(eyre!("Value not found in storage."))?;
+    let value = storage_value.to_string().highlight();
 
-    let value = decoded_value_thunk.to_value()?.to_string().highlight();
     writedoc! {output, "
 
     The value of the storage entry is:

cli/src/commands/explore/pallets/storage.rs

@@ -101,15 +101,13 @@ pub async fn run<'a>(
             return format!("The method does not require an {input_value_placeholder}");
         }
 
-        let fields: Vec<(Option<&str>, u32)> = method
-            .inputs()
-            .map(|f| (Some(f.name.as_str()), f.ty))
-            .collect();
+        let fields: Vec<(Option<&str>, u32)> =
+            method.inputs().map(|f| (Some(&*f.name), f.id)).collect();
         let fields_description =
             fields_description(&fields, method.name(), metadata.types()).indent(4);
 
         let fields_example =
-            fields_composite_example(method.inputs().map(|e| e.ty), metadata.types())
+            fields_composite_example(method.inputs().map(|e| e.id), metadata.types())
                 .indent(4)
                 .highlight();
 
@@ -164,13 +162,14 @@ pub async fn run<'a>(
             {value_str}
             "}?;
             // encode, then decode. This ensures that the scale value is of the correct shape for the param:
-            let bytes = value.encode_as_type(ty.ty, metadata.types())?;
-            let value = Value::decode_as_type(&mut &bytes[..], ty.ty, metadata.types())?;
+            let bytes = value.encode_as_type(ty.id, metadata.types())?;
+            let value = Value::decode_as_type(&mut &bytes[..], ty.id, metadata.types())?;
             Ok(value)
         })
         .collect::<color_eyre::Result<Vec<Value>>>()?;
 
-    let method_call = subxt::dynamic::runtime_api_call(api_name, method.name(), args_data);
+    let method_call =
+        subxt::dynamic::runtime_api_call::<_, Value>(api_name, method.name(), args_data);
     let client = create_client(&file_or_url).await?;
     let output_value = client
         .runtime_api()
@@ -179,7 +178,7 @@ pub async fn run<'a>(
         .call(method_call)
         .await?;
 
-    let output_value = output_value.to_value()?.to_string().highlight();
+    let output_value = output_value.to_string().highlight();
     writedoc! {output, "
 
     Returned value:

cli/src/commands/explore/runtime_apis/mod.rs

@@ -1,3 +1,3 @@
 # result_large_err lint complains if error variant is 128 bytes or more by default.
 # Our error is. Let's up this limit a bit for now to avoid lots of warnings.
-large-error-threshold = 256
+large-error-threshold = 512

clippy.toml

@@ -57,16 +57,17 @@ fn generate_custom_value_fn(
         .types()
         .resolve(custom_value.type_id())
         .is_some();
+
     let (return_ty, decodable) = if type_is_valid {
         let return_ty = type_gen
             .resolve_type_path(custom_value.type_id())
             .expect("type is in metadata; qed")
             .to_token_stream(type_gen.settings());
-        let decodable = quote!(#crate_path::utils::Yes);
+        let decodable = quote!(#crate_path::utils::Maybe);
         (return_ty, decodable)
     } else {
         // if type registry does not contain the type, we can just return the Encoded scale bytes.
-        (quote!(()), quote!(()))
+        (quote!(()), quote!(#crate_path::utils::No))
     };
 
     Some(quote!(