Publish indexed transactions with BLAKE2b hashes to IPFS DHT (#10468)

Add `--ipfs-bootnodes` flag for specifying IPFS bootnodes. If passed
along with `--ipfs-server`, the node will register as a content provider
in IPFS DHT of indexed transactions with BLAKE2b hashes of the last two
weeks (or pruning depth if smaller).

## Follow-ups
- Support other hashes (sha2-256 specifically) and CID codecs
- Adjust `IPFS_MAX_BLOCKS` for chains with elastic scaling
- Speedup DHT publishing in litep2p (should aim at 10s single provider
publish time)

---------

Co-authored-by: cmd[bot] <41898282+github-actions[bot]@users.noreply.github.com>

Author: dmitry-markin

Cargo.lock

@@ -24,6 +24,7 @@ sc-consensus = { workspace = true, default-features = true }
 sc-network = { workspace = true, default-features = true }
 sp-api = { workspace = true, default-features = true }
 sp-consensus = { workspace = true, default-features = true }
+sp-core = { workspace = true, default-features = true }
 sp-maybe-compressed-blob = { workspace = true, default-features = true }
 sp-runtime = { workspace = true, default-features = true }
 sp-version = { workspace = true, default-features = true }

cumulus/client/pov-recovery/Cargo.toml

@@ -42,6 +42,7 @@ use sc_consensus::import_queue::RuntimeOrigin;
 use sc_utils::mpsc::{TracingUnboundedReceiver, TracingUnboundedSender};
 use sp_api::RuntimeApiInfo;
 use sp_blockchain::Info;
+use sp_core::H256;
 use sp_runtime::{generic::SignedBlock, Justifications};
 use sp_version::RuntimeVersion;
 use std::{
@@ -198,11 +199,15 @@ impl<Block: BlockT> BlockBackend<Block> for ParachainClient<Block> {
 		unimplemented!()
 	}
 
-	fn indexed_transaction(&self, _: Block::Hash) -> sp_blockchain::Result<Option<Vec<u8>>> {
+	fn indexed_transaction(&self, _: H256) -> sp_blockchain::Result<Option<Vec<u8>>> {
 		unimplemented!()
 	}
 
-	fn has_indexed_transaction(&self, _: Block::Hash) -> sp_blockchain::Result<bool> {
+	fn has_indexed_transaction(&self, _: H256) -> sp_blockchain::Result<bool> {
+		unimplemented!()
+	}
+
+	fn block_indexed_hashes(&self, _: Block::Hash) -> sp_blockchain::Result<Option<Vec<H256>>> {
 		unimplemented!()
 	}
 

cumulus/client/pov-recovery/src/tests.rs

@@ -76,7 +76,7 @@ pub(crate) fn build_collator_network<Network: NetworkBackend<Block, Hash>>(
 		protocol_id,
 		metrics_registry: config.prometheus_config.as_ref().map(|config| config.registry.clone()),
 		block_announce_config,
-		bitswap_config: None,
+		ipfs_config: None,
 		notification_metrics,
 	};
 

cumulus/client/relay-chain-minimal-node/src/network.rs

@@ -0,0 +1,28 @@
+title: Publish indexed transactions with BLAKE2b hashes to IPFS DHT
+doc:
+- audience: [Node Dev, Node Operator]
+  description: Add `--ipfs-bootnodes` flag for specifying IPFS bootnodes. If passed
+    along with `--ipfs-server`, the node will register as a content provider in IPFS
+    DHT of indexed transactions with BLAKE2b hashes of the last two weeks (or pruning
+    depth if smaller).
+crates:
+- name: sc-network
+  bump: major
+- name: sc-network-types
+  bump: minor
+- name: cumulus-client-pov-recovery
+  bump: minor
+- name: sc-client-api
+  bump: major
+- name: sc-client-db
+  bump: minor
+- name: sc-rpc-spec-v2
+  bump: patch
+- name: sc-service
+  bump: major
+- name: sp-blockchain
+  bump: minor
+- name: cumulus-relay-chain-minimal-node
+  bump: patch
+- name: sc-cli
+  bump: major

prdoc/pr_10468.prdoc

@@ -19,7 +19,7 @@
 //! A set of APIs supported by the client along with their primitives.
 
 use sp_consensus::BlockOrigin;
-use sp_core::storage::StorageKey;
+use sp_core::{storage::StorageKey, H256};
 use sp_runtime::{
 	generic::SignedBlock,
 	traits::{Block as BlockT, NumberFor},
@@ -132,13 +132,19 @@ pub trait BlockBackend<Block: BlockT> {
 		hash: Block::Hash,
 	) -> sp_blockchain::Result<Option<Vec<<Block as BlockT>::Extrinsic>>>;
 
-	/// Get all indexed transactions for a block,
-	/// including renewed transactions.
+	/// Get all indexed transactions for a block, including renewed transactions.
 	///
-	/// Note that this will only fetch transactions
-	/// that are indexed by the runtime with `storage_index_transaction`.
+	/// Note that this will only fetch transactions that are indexed by the runtime with
+	/// `storage_index_transaction`.
 	fn block_indexed_body(&self, hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<Vec<u8>>>>;
 
+	/// Get the BLAKE2b-256 hashes of all indexed transactions in a block, including renewed
+	/// transactions.
+	///
+	/// Note that this will only fetch transactions that are indexed by the runtime with
+	/// `storage_index_transaction`.
+	fn block_indexed_hashes(&self, hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<H256>>>;
+
 	/// Get full block by hash.
 	fn block(&self, hash: Block::Hash) -> sp_blockchain::Result<Option<SignedBlock<Block>>>;
 
@@ -151,14 +157,14 @@ pub trait BlockBackend<Block: BlockT> {
 	/// Get block hash by number.
 	fn block_hash(&self, number: NumberFor<Block>) -> sp_blockchain::Result<Option<Block::Hash>>;
 
-	/// Get single indexed transaction by content hash.
+	/// Get single indexed transaction by content hash (BLAKE2b-256).
 	///
 	/// Note that this will only fetch transactions
 	/// that are indexed by the runtime with `storage_index_transaction`.
-	fn indexed_transaction(&self, hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<u8>>>;
+	fn indexed_transaction(&self, hash: H256) -> sp_blockchain::Result<Option<Vec<u8>>>;
 
-	/// Check if transaction index exists.
-	fn has_indexed_transaction(&self, hash: Block::Hash) -> sp_blockchain::Result<bool> {
+	/// Check if transaction index exists given its BLAKE2b-256 hash.
+	fn has_indexed_transaction(&self, hash: H256) -> sp_blockchain::Result<bool> {
 		Ok(self.indexed_transaction(hash)?.is_some())
 	}
 

substrate/client/api/src/client.rs

@@ -21,7 +21,7 @@
 use parking_lot::RwLock;
 use sp_blockchain::{CachedHeaderMetadata, HeaderMetadata};
 use sp_core::{
-	offchain::storage::InMemOffchainStorage as OffchainStorage, storage::well_known_keys,
+	offchain::storage::InMemOffchainStorage as OffchainStorage, storage::well_known_keys, H256,
 };
 use sp_runtime::{
 	generic::BlockId,
@@ -427,7 +427,11 @@ impl<Block: BlockT> blockchain::Backend<Block> for Blockchain<Block> {
 		unimplemented!()
 	}
 
-	fn indexed_transaction(&self, _hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<u8>>> {
+	fn indexed_transaction(&self, _hash: H256) -> sp_blockchain::Result<Option<Vec<u8>>> {
+		unimplemented!("Not supported by the in-mem backend.")
+	}
+
+	fn block_indexed_hashes(&self, _hash: Block::Hash) -> sp_blockchain::Result<Option<Vec<H256>>> {
 		unimplemented!("Not supported by the in-mem backend.")
 	}
 

substrate/client/api/src/in_mem.rs

@@ -153,6 +153,10 @@ pub struct NetworkParams {
 	#[arg(long)]
 	pub ipfs_server: bool,
 
+	/// Specify a list of IPFS bootstrap nodes.
+	#[arg(long, value_name = "ADDR", num_args = 1.., requires = "ipfs_server")]
+	pub ipfs_bootnodes: Vec<MultiaddrWithPeerId>,
+
 	/// Blockchain syncing mode.
 	#[arg(
 		long,
@@ -284,6 +288,7 @@ impl NetworkParams {
 			kademlia_disjoint_query_paths: self.kademlia_disjoint_query_paths,
 			kademlia_replication_factor: self.kademlia_replication_factor,
 			ipfs_server: self.ipfs_server,
+			ipfs_bootnodes: self.ipfs_bootnodes.clone(),
 			sync_mode: self.sync.into(),
 			network_backend: self.network_backend.into(),
 		}

substrate/client/cli/src/params/network_params.rs

@@ -697,6 +697,30 @@ impl<Block: BlockT> BlockchainDb<Block> {
 		}
 		Ok(None)
 	}
+
+	fn block_indexed_hashes_iter(
+		&self,
+		hash: Block::Hash,
+	) -> ClientResult<Option<impl Iterator<Item = DbHash>>> {
+		let Some(body) = read_db(
+			&*self.db,
+			columns::KEY_LOOKUP,
+			columns::BODY_INDEX,
+			BlockId::<Block>::Hash(hash),
+		)?
+		else {
+			return Ok(None);
+		};
+		match Vec::<DbExtrinsic<Block>>::decode(&mut &body[..]) {
+			Ok(index) => Ok(Some(index.into_iter().flat_map(|ex| match ex {
+				DbExtrinsic::Indexed { hash, .. } => Some(hash),
+				_ => None,
+			}))),
+			Err(err) => {
+				Err(sp_blockchain::Error::Backend(format!("Error decoding body list: {err}")))
+			},
+		}
+	}
 }
 
 impl<Block: BlockT> sc_client_api::blockchain::HeaderBackend<Block> for BlockchainDb<Block> {
@@ -782,44 +806,32 @@ impl<Block: BlockT> sc_client_api::blockchain::Backend<Block> for BlockchainDb<B
 		children::read_children(&*self.db, columns::META, meta_keys::CHILDREN_PREFIX, parent_hash)
 	}
 
-	fn indexed_transaction(&self, hash: Block::Hash) -> ClientResult<Option<Vec<u8>>> {
+	fn indexed_transaction(&self, hash: DbHash) -> ClientResult<Option<Vec<u8>>> {
 		Ok(self.db.get(columns::TRANSACTION, hash.as_ref()))
 	}
 
-	fn has_indexed_transaction(&self, hash: Block::Hash) -> ClientResult<bool> {
+	fn has_indexed_transaction(&self, hash: DbHash) -> ClientResult<bool> {
 		Ok(self.db.contains(columns::TRANSACTION, hash.as_ref()))
 	}
 
+	fn block_indexed_hashes(&self, hash: Block::Hash) -> ClientResult<Option<Vec<DbHash>>> {
+		self.block_indexed_hashes_iter(hash).map(|hashes| hashes.map(Iterator::collect))
+	}
+
 	fn block_indexed_body(&self, hash: Block::Hash) -> ClientResult<Option<Vec<Vec<u8>>>> {
-		let body = match read_db(
-			&*self.db,
-			columns::KEY_LOOKUP,
-			columns::BODY_INDEX,
-			BlockId::<Block>::Hash(hash),
-		)? {
-			Some(body) => body,
-			None => return Ok(None),
-		};
-		match Vec::<DbExtrinsic<Block>>::decode(&mut &body[..]) {
-			Ok(index) => {
-				let mut transactions = Vec::new();
-				for ex in index.into_iter() {
-					if let DbExtrinsic::Indexed { hash, .. } = ex {
-						match self.db.get(columns::TRANSACTION, hash.as_ref()) {
-							Some(t) => transactions.push(t),
-							None => {
-								return Err(sp_blockchain::Error::Backend(format!(
-									"Missing indexed transaction {hash:?}",
-								)))
-							},
-						}
-					}
-				}
-				Ok(Some(transactions))
-			},
-			Err(err) => {
-				Err(sp_blockchain::Error::Backend(format!("Error decoding body list: {err}")))
-			},
+		match self.block_indexed_hashes_iter(hash) {
+			Ok(Some(hashes)) => Ok(Some(
+				hashes
+					.map(|hash| match self.db.get(columns::TRANSACTION, hash.as_ref()) {
+						Some(t) => Ok(t),
+						None => Err(sp_blockchain::Error::Backend(format!(
+							"Missing indexed transaction {hash:?}",
+						))),
+					})
+					.collect::<Result<_, _>>()?,
+			)),
+			Ok(None) => Ok(None),
+			Err(err) => Err(err),
 		}
 	}
 }

substrate/client/db/src/lib.rs

@@ -30,6 +30,7 @@ async-channel = { workspace = true }
 async-trait = { workspace = true }
 asynchronous-codec = { workspace = true }
 bytes = { workspace = true, default-features = true }
+cid = { workspace = true }
 codec = { features = ["derive"], workspace = true, default-features = true }
 either = { workspace = true, default-features = true }
 fnv = { workspace = true }
@@ -69,7 +70,6 @@ zeroize = { workspace = true, default-features = true }
 
 [dev-dependencies]
 assert_matches = { workspace = true }
-cid = { workspace = true }
 multistream-select = { workspace = true }
 sc-block-builder = { workspace = true, default-features = true }
 sp-consensus = { workspace = true, default-features = true }