Add subxt-historic crate for accesing historic (non head-of-chain) blocks (#2040)

* WIP subxt-historic

* WIP subxt-historic

* WIP subxt-historic; flesh out basic foundations

* WIP filling in extrinsic decoding functionality

* iter and decode transaction extensions

* Fill in the Online/OfflineClient APIs and move more things to be part of the chain Config

* WIP storage

* clippy, fmt, finish extrinsics example

* prep for 0.0.1 release to claim crate name

* fix README link

* fmt

* WIP thinking about storage APIs

* WIP working out storage APIs

* Storage plain value fetching first pass

* WIP storage: first pass iterating over values done

* First apss finishing storage APIs

* fmt and clippy

* Create a storage example showing fetch and iteration

* Bump to frame-decode 0.9.0

* Bump subxt-historic to 0.0.3 for preview release

* Remove unused deps

* fix import

* clippy

* doc fixes

* tweak CI and fix some cargo hack findings

* Update README: subxt-historic is prerelease

Author: jsdw

.github/workflows/rust.yml

@@ -187,6 +187,9 @@ jobs:
       - name: Cargo hack; check each subxt feature
         run: cargo hack -p subxt --each-feature check --exclude-no-default-features --exclude-all-features --exclude-features web --features native
 
+      - name: Cargo hack; check each subxt feature
+        run: cargo hack -p subxt-historic --each-feature check --exclude-no-default-features --exclude-all-features --exclude-features web --features native
+
       # Subxt-signer has the "subxt" features enabled in the "check all targets" test. Run it on its own to
       # check it without. We can't enable subxt or web features here, so no cargo hack.
       - name: Cargo check subxt-signer
@@ -213,7 +216,7 @@ jobs:
 
       # Next, check each other package in isolation.
       - name: Cargo hack; check each feature/crate on its own
-        run: cargo hack --exclude subxt --exclude subxt-signer --exclude subxt-lightclient --exclude subxt-rpcs --exclude-all-features --each-feature check --workspace
+        run: cargo hack --exclude subxt --exclude subxt-historic --exclude subxt-signer --exclude subxt-lightclient --exclude subxt-rpcs --exclude-all-features --each-feature check --workspace
 
       # Check the full examples, which aren't a part of the workspace so are otherwise ignored.
       - name: Cargo check parachain-example

Cargo.lock

@@ -4,6 +4,7 @@ members = [
     "codegen",
     "core",
     "lightclient",
+    "historic",
     "testing/substrate-runner",
     "testing/test-runtime",
     "testing/integration-tests",
@@ -80,8 +81,8 @@ darling = "0.20.10"
 derive-where = "1.2.7"
 either = { version = "1.13.0", default-features = false }
 finito = { version = "0.1.0", default-features = false }
-frame-decode = { version = "0.8.0", default-features = false }
-frame-metadata = { version = "23.0.0", default-features = false, features = ["unstable"] }
+frame-decode = { version = "0.9.0", default-features = false }
+frame-metadata = { version = "23.0.0", default-features = false }
 futures = { version = "0.3.31", default-features = false, features = ["std"] }
 getrandom = { version = "0.2", default-features = false }
 hashbrown = "0.14.5"
@@ -101,6 +102,8 @@ scale-value = { version = "0.18.0", default-features = false }
 scale-bits = { version = "0.7.0", default-features = false }
 scale-decode = { version = "0.16.0", default-features = false }
 scale-encode = { version = "0.10.0", default-features = false }
+scale-type-resolver = { version = "0.2.0" }
+scale-info-legacy = { version = "0.2.3" }
 scale-typegen = "0.11.1"
 scale-typegen-description = "0.11.0"
 serde = { version = "1.0.210", default-features = false, features = ["derive"] }

Cargo.toml

@@ -0,0 +1,63 @@
+[package]
+name = "subxt-historic"
+version = "0.0.3"
+authors.workspace = true
+edition.workspace = true
+rust-version.workspace = true
+publish = true
+
+license.workspace = true
+readme = "README.md"
+repository.workspace = true
+documentation.workspace = true
+homepage.workspace = true
+description = "Download non head-of-chain blocks and state from Substrate based nodes"
+keywords = ["parity", "substrate", "blockchain"]
+
+[lints]
+workspace = true
+
+[features]
+default = ["jsonrpsee", "native"]
+
+# Enable this for native (ie non web/wasm builds).
+# Exactly 1 of "web" and "native" is expected.
+native = [
+    "subxt-rpcs/native",
+]
+
+# Enable this for web/wasm builds.
+# Exactly 1 of "web" and "native" is expected.
+web = [
+    "subxt-rpcs/web",
+]
+
+# Enable this to use the reconnecting rpc client
+reconnecting-rpc-client = ["subxt-rpcs/reconnecting-rpc-client"]
+
+# Enable this to use jsonrpsee, which enables the jsonrpsee RPC client, and
+# a couple of util functions which rely on jsonrpsee.
+jsonrpsee = [
+    "subxt-rpcs/jsonrpsee",
+]
+
+[dependencies]
+subxt-rpcs = { workspace = true }
+frame-decode = { workspace = true, features = ["legacy", "legacy-types"] }
+frame-metadata = { workspace = true, features = ["std", "legacy"] }
+scale-type-resolver = { workspace = true }
+codec = { workspace = true }
+primitive-types = { workspace = true }
+scale-info = { workspace = true }
+scale-info-legacy = { workspace = true }
+scale-decode = { workspace = true }
+thiserror = { workspace = true }
+sp-crypto-hashing = { workspace = true }
+url = { workspace = true }
+futures = { workspace = true }
+
+[dev-dependencies]
+tokio = { workspace = true, features = ["full"] }
+scale-value = { workspace = true }
+scale-decode = { workspace = true, features = ["derive"] }
+hex = { workspace = true }

historic/Cargo.toml

@@ -0,0 +1,16 @@
+# subxt-historic
+
+**This create is a work in progress and currently is released only as a preview.**
+
+While `subxt` is a library for working at the head of a chain (submitting transactions and obtaining the current state), `subxt-historic` is a library for decoding blocks and state that are anywhere in a chain. To broadly summarize the differences:
+
+| Feature                                 | subxt                        | subxt-historic                |
+|-----------------------------------------|------------------------------|-------------------------------|
+| Block access                            | Head of chain                | Any block in chain            |
+| Connection to chain                     | Light client or RPC node     | Archive RPC nodes only        |
+| Transaction submission                  | Yes                          | No                            |
+| Metadata compatibility                  | V14 and newer                | Any version                   |
+
+# Examples
+
+See the [examples](https://github.com/paritytech/subxt/tree/master/historic/examples) folder for examples of how to use `subxt-historic`.

historic/README.md

@@ -0,0 +1,83 @@
+#![allow(missing_docs)]
+use subxt_historic::{Error, OnlineClient, PolkadotConfig};
+
+#[tokio::main]
+async fn main() -> Result<(), Error> {
+    // Configuration for the Polkadot relay chain.
+    let config = PolkadotConfig::new();
+
+    // Create an online client for the Polkadot relay chain, pointed at a Polkadot archive node.
+    let client = OnlineClient::from_url(config, "wss://rpc.polkadot.io").await?;
+
+    // Iterate through some randomly selected old blocks to show how to fetch and decode extrinsics.
+    for block_number in 123456.. {
+        println!("=== Block {block_number} ===");
+
+        // Point the client at a specific block number. By default this will download and cache
+        // metadata for the required spec version (so it's cheaper to instantiate again), if it
+        // hasn't already, and borrow the relevant legacy types from the client.
+        let client_at_block = client.at(block_number).await?;
+
+        // Fetch the extrinsics at that block.
+        let extrinsics = client_at_block.extrinsics().fetch().await?;
+
+        // Now, we have various operations to work with them. Here we print out various details
+        // about each extrinsic.
+        for extrinsic in extrinsics.iter() {
+            println!(
+                "{}.{}",
+                extrinsic.call().pallet_name(),
+                extrinsic.call().name()
+            );
+
+            if let Some(signature) = extrinsic.signature_bytes() {
+                println!("  Signature: 0x{}", hex::encode(signature));
+            }
+
+            println!("  Call Data:");
+
+            // We can decode each of the fields (in this example we decode everything into a
+            // scale_value::Value type, which can represent any SCALE encoded data, but if you
+            // have an idea of the type then you can try to decode into that type instead):
+            for field in extrinsic.call().fields().iter() {
+                println!(
+                    "    {}: {}",
+                    field.name(),
+                    field.decode::<scale_value::Value>().unwrap()
+                );
+            }
+
+            // Or, all of them at once:
+            println!(
+                "    All: {}",
+                extrinsic
+                    .call()
+                    .fields()
+                    .decode::<scale_value::Composite<_>>()
+                    .unwrap()
+            );
+
+            // We can also look at things like the transaction extensions:
+            if let Some(extensions) = extrinsic.transaction_extensions() {
+                println!("  Transaction Extensions:");
+
+                // We can decode each of them:
+                for extension in extensions.iter() {
+                    println!(
+                        "    {}: {}",
+                        extension.name(),
+                        extension.decode::<scale_value::Value>().unwrap()
+                    );
+                }
+
+                // Or all of them at once:
+                println!(
+                    "    All: {}",
+                    extensions.decode::<scale_value::Composite<_>>().unwrap()
+                );
+            }
+        }
+    }
+
+    Ok(())
+}